mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d2bf06ac7 | ||
|
|
b8faaf250f | ||
|
|
40137b1205 | ||
|
|
c67cde4e90 | ||
|
|
2e9213a784 | ||
|
|
87496b5656 | ||
|
|
c856e4306f | ||
|
|
92090f28bf | ||
|
|
12c2b612de | ||
|
|
e8af0b9c66 | ||
|
|
80776e8e03 | ||
|
|
b296c795ac | ||
|
|
9fc162a82f | ||
|
|
9532bd438b | ||
|
|
73319f1943 | ||
|
|
ee56d6a1d2 | ||
|
|
5d29ddfffc | ||
|
|
45903b684b | ||
|
|
a580874697 | ||
|
|
885369f19f | ||
|
|
24ff339c97 | ||
|
|
e582eb73d2 | ||
|
|
ec252b3b46 | ||
|
|
6503f36750 | ||
|
|
2366f66f33 | ||
|
|
cc0303c65e | ||
|
|
b11eeac29e | ||
|
|
ad5f3ba997 | ||
|
|
15455ca619 | ||
|
|
904315c209 | ||
|
|
95d720dd86 | ||
|
|
74f59cc4f8 | ||
|
|
e979454457 | ||
|
|
0af8610a48 | ||
|
|
8226089ade | ||
|
|
1bd40ffa67 | ||
|
|
34a6ccce43 | ||
|
|
c275119f12 | ||
|
|
9322b7aeb8 | ||
|
|
c6444b2bf1 | ||
|
|
55075677eb | ||
|
|
733aa5f88a | ||
|
|
d704910857 | ||
|
|
ad6c21392f | ||
|
|
55e87c3b16 | ||
|
|
0c4da6c838 | ||
|
|
225c1166c6 | ||
|
|
97b91d3409 | ||
|
|
c953f6f44d | ||
|
|
d6b9fedbc7 | ||
|
|
720eb0c227 | ||
|
|
495c19b5e4 | ||
|
|
237ba96233 | ||
|
|
ae46d89de6 | ||
|
|
69f73e8c5d | ||
|
|
245f87ef35 | ||
|
|
81e51db0b6 | ||
|
|
4634496638 | ||
|
|
4e2f0a9346 | ||
|
|
304b3d4772 | ||
|
|
74629f3ef2 | ||
|
|
c9d48188af | ||
|
|
7e7b88a59e | ||
|
|
5c66dc198d | ||
|
|
47efce7a90 | ||
|
|
ea4d3f0ba7 | ||
|
|
2d03e27dcf | ||
|
|
cf7ff80f71 | ||
|
|
93ab75e42b | ||
|
|
449dc7da79 | ||
|
|
3b129a483a | ||
|
|
3e0d60bb1f | ||
|
|
1d3645fc26 | ||
|
|
18bff86214 | ||
|
|
114084107c | ||
|
|
d60f7238da | ||
|
|
ea684dceeb | ||
|
|
dfd496b41a | ||
|
|
6125b10276 | ||
|
|
071271f003 | ||
|
|
13a9df2e07 | ||
|
|
37a5a6702b | ||
|
|
afb68e97d6 | ||
|
|
93b9741af2 | ||
|
|
013d592454 | ||
|
|
0f56e9b80c | ||
|
|
6ec5ecb9ac | ||
|
|
460d439351 | ||
|
|
f39406dd80 | ||
|
|
9de8e75238 | ||
|
|
fee34b6bd1 | ||
|
|
cbe8699322 | ||
|
|
066e5551f8 | ||
|
|
1d73b736a9 | ||
|
|
8ae6635b9c | ||
|
|
574f0d580e | ||
|
|
0879515ee8 | ||
|
|
be40783454 |
@@ -40,7 +40,7 @@ jobs:
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -61,9 +61,44 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
# PR 场景需要 base 提交做冻结清单的 append-only 比对。
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && 0 || 1 }}
|
||||
|
||||
# 已应用迁移的字节被 sqlx 启动校验钉死:改动已发布迁移会让所有
|
||||
# 存量部署重启即失败。冻结清单保证旧文件不被改、新文件必须入册。
|
||||
- name: Migration freeze check
|
||||
working-directory: backend/migrations
|
||||
shell: bash
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sha256sum --check --strict .frozen-sha256
|
||||
for file in *.sql; do
|
||||
if ! awk '{print $2}' .frozen-sha256 | grep -Fxq "$file"; then
|
||||
echo "::error file=backend/migrations/$file::migration $file is not frozen; append its sha256 to backend/migrations/.frozen-sha256"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# 清单自证不足以防「改迁移 + 同 PR 刷新清单行」:校验对新哈希通过。
|
||||
# 对 PR,断言 .frozen-sha256 相对 base 只增不改——base 里的每一行
|
||||
# 必须原样保留(改哈希=删行,删迁移=删行,都会被挡)。
|
||||
if [ -n "${BASE_SHA:-}" ]; then
|
||||
base_manifest="$(git show "${BASE_SHA}:backend/migrations/.frozen-sha256" 2>/dev/null || true)"
|
||||
if [ -n "$base_manifest" ]; then
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
if ! grep -Fxq "$line" .frozen-sha256; then
|
||||
file="$(printf '%s' "$line" | awk '{print $2}')"
|
||||
echo "::error file=backend/migrations/.frozen-sha256::frozen migration $file was modified or removed; released migrations are append-only — add a new numbered migration instead"
|
||||
exit 1
|
||||
fi
|
||||
done <<< "$base_manifest"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
@@ -117,7 +152,7 @@ jobs:
|
||||
CI: 'true'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
@@ -156,7 +191,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
container: ${{ steps.filter.outputs.container }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -29,6 +29,7 @@ jobs:
|
||||
outputs:
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
docker_version: ${{ steps.version.outputs.docker_version }}
|
||||
build_time: ${{ steps.version.outputs.build_time }}
|
||||
git_sha: ${{ steps.version.outputs.git_sha }}
|
||||
platform_matrix: ${{ steps.version.outputs.platform_matrix }}
|
||||
@@ -40,7 +41,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
fetch-depth: 0
|
||||
@@ -104,7 +105,7 @@ jobs:
|
||||
require_string(platform, key, "#{platforms_path}: platforms[#{index}]")
|
||||
end
|
||||
archive = platform["archive"]
|
||||
fail("#{platforms_path}: platforms[#{index}].archive must be tar.gz or zip") unless ["tar.gz", "zip"].include?(archive)
|
||||
fail("#{platforms_path}: platforms[#{index}].archive must be tar.gz") unless archive == "tar.gz"
|
||||
platform_key = "#{platform["os"]}-#{platform["arch"]}"
|
||||
fail("#{platforms_path}: duplicate platform #{platform_key}") if seen_platforms[platform_key]
|
||||
seen_platforms[platform_key] = true
|
||||
@@ -146,6 +147,8 @@ jobs:
|
||||
|
||||
puts "tag=#{release_tag}"
|
||||
puts "version=#{version}"
|
||||
# Docker tags forbid "+", so semver build metadata maps to "-" in image tags.
|
||||
puts "docker_version=#{version.tr("+", "-")}"
|
||||
puts "build_time=#{Time.now.utc.iso8601}"
|
||||
puts "git_sha=#{git_sha.strip}"
|
||||
puts "platform_matrix=#{matrix}"
|
||||
@@ -174,7 +177,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -234,7 +237,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -286,7 +289,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -321,7 +324,6 @@ jobs:
|
||||
import pathlib
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
|
||||
APP_NAME = "codex-proxy-rs"
|
||||
|
||||
@@ -357,8 +359,7 @@ jobs:
|
||||
|
||||
binary_dest = package_root / platform["binary"]
|
||||
shutil.copy2(binary_src, binary_dest)
|
||||
if platform["os"] != "windows":
|
||||
binary_dest.chmod(0o755)
|
||||
binary_dest.chmod(0o755)
|
||||
shutil.copytree(frontend_dir, web_dist, dirs_exist_ok=True)
|
||||
|
||||
archive_name = f"{APP_NAME}_{version}_{platform['os']}_{platform['arch']}.{platform['archive']}"
|
||||
@@ -366,11 +367,6 @@ jobs:
|
||||
if platform["archive"] == "tar.gz":
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.add(package_root, arcname=APP_NAME)
|
||||
elif platform["archive"] == "zip":
|
||||
with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for path in sorted(package_root.rglob("*")):
|
||||
if path.is_file():
|
||||
archive.write(path, path.relative_to(package_dir))
|
||||
else:
|
||||
raise SystemExit(f"Unsupported archive format: {platform['archive']}")
|
||||
archives.append(archive_path)
|
||||
@@ -412,7 +408,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -458,7 +454,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -512,7 +508,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
@@ -529,7 +525,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -539,7 +535,7 @@ jobs:
|
||||
id: manifest
|
||||
shell: bash
|
||||
env:
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
DOCKER_VERSION: ${{ needs.release-info.outputs.docker_version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
DOCKER_PLATFORMS: ${{ needs.release-info.outputs.docker_platforms }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
@@ -562,7 +558,7 @@ jobs:
|
||||
fi
|
||||
|
||||
target_args=(
|
||||
-t "${image}:${VERSION}"
|
||||
-t "${image}:${DOCKER_VERSION}"
|
||||
-t "${image}:sha-${GIT_SHA}"
|
||||
)
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
@@ -570,7 +566,7 @@ jobs:
|
||||
fi
|
||||
docker buildx imagetools create "${target_args[@]}" "${sources[@]}"
|
||||
|
||||
inspect_output="$(docker buildx imagetools inspect "${image}:${VERSION}")"
|
||||
inspect_output="$(docker buildx imagetools inspect "${image}:${DOCKER_VERSION}")"
|
||||
platform_list="$(awk '$1 == "Platform:" { print $2 }' <<< "${inspect_output}")"
|
||||
for platform in "${platforms[@]}"; do
|
||||
if ! grep -Eq "^${platform}(/|$)" <<< "${platform_list}"; then
|
||||
@@ -580,7 +576,7 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
digest="$(docker buildx imagetools inspect "${image}:${VERSION}" --format '{{ .Manifest.Digest }}')"
|
||||
digest="$(docker buildx imagetools inspect "${image}:${DOCKER_VERSION}" --format '{{ .Manifest.Digest }}')"
|
||||
if [ -z "${digest}" ]; then
|
||||
echo "missing image digest"
|
||||
exit 1
|
||||
@@ -598,7 +594,7 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
DIGEST: ${{ steps.manifest.outputs.digest }}
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
DOCKER_VERSION: ${{ needs.release-info.outputs.docker_version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
run: |
|
||||
@@ -609,7 +605,7 @@ jobs:
|
||||
fi
|
||||
|
||||
image="${REGISTRY}/${IMAGE_NAME}"
|
||||
tags=("${image}:${VERSION}" "${image}:sha-${GIT_SHA}")
|
||||
tags=("${image}:${DOCKER_VERSION}" "${image}:sha-${GIT_SHA}")
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
tags+=("${image}:latest")
|
||||
fi
|
||||
@@ -629,12 +625,15 @@ jobs:
|
||||
prerelease: ${{ needs.release-info.outputs.prerelease == 'true' }}
|
||||
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
|
||||
body: |
|
||||
> [!WARNING]
|
||||
> **Upgrading across major versions?** Database schema and configuration are only compatible within a major version, and the one-click self-update refuses cross-major targets (releases before 3.0 lack this gate — do not use their built-in update to reach a newer major). Deploy the new major fresh (new `.runtime/` data directory) and re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
|
||||
## 📥 Installation
|
||||
|
||||
**Docker:**
|
||||
```bash
|
||||
# GitHub Container Registry
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.docker_version }}
|
||||
```
|
||||
|
||||
**Manual download:**
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 保持 Responses 语义透明;xAI 的协议适配只驻留在 `providers/xai` |
|
||||
| 透明边界 | OpenAI SSE 与 WebSocket 均原样转发上游字节;xAI 在 `providers/xai` 内做 Grok 与 Responses 协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因 |
|
||||
@@ -91,7 +91,7 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
2. 在账号页选择对应 Provider:OpenAI 支持 OAuth、账号文件与 Agent Identity;xAI 支持 OAuth
|
||||
和兼容账号文件导入。
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置模型范围、速率与并发限制。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
@@ -158,9 +158,15 @@ docker compose -f deploy/compose.yaml up -d
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点和自更新状态。删除该目录会永久清除
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点密钥和自更新状态。删除该目录会永久清除
|
||||
> 运行状态。
|
||||
|
||||
> [!WARNING]
|
||||
> 跨大版本不是原地升级:数据库 schema 与配置只在同一大版本内兼容,内置一键自更新
|
||||
> 会拒绝跨大版本目标(3.0 之前的旧版没有这道门禁,不要用旧版界面的一键更新跨大版本)。
|
||||
> 升级到新的大版本需要全新部署(全新 `.runtime/` 数据目录),并重新导入或重新授权
|
||||
> Provider 账号与客户端 Key。
|
||||
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
|
||||
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
|
||||
|
||||
Generated
+126
-62
@@ -142,7 +142,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -208,7 +208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
@@ -275,6 +275,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -444,7 +450,7 @@ dependencies = [
|
||||
"provider-openai",
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"syn 2.0.118",
|
||||
"syn 3.0.3",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -669,7 +675,7 @@ dependencies = [
|
||||
"aead",
|
||||
"blake2",
|
||||
"crypto_secretbox",
|
||||
"curve25519-dalek",
|
||||
"curve25519-dalek 4.1.3",
|
||||
"salsa20",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -708,8 +714,23 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"curve25519-dalek-derive",
|
||||
"digest 0.10.7",
|
||||
"fiat-crypto",
|
||||
"fiat-crypto 0.2.9",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
"curve25519-dalek-derive",
|
||||
"digest 0.11.3",
|
||||
"fiat-crypto 0.3.0",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -723,7 +744,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -761,6 +782,16 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
@@ -799,7 +830,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -829,34 +860,35 @@ version = "0.16.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
|
||||
dependencies = [
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
"digest 0.10.7",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature",
|
||||
"spki",
|
||||
"signature 2.2.0",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ed25519"
|
||||
version = "2.2.3"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
|
||||
checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a"
|
||||
dependencies = [
|
||||
"pkcs8",
|
||||
"signature",
|
||||
"pkcs8 0.11.0",
|
||||
"signature 3.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ed25519-dalek"
|
||||
version = "2.2.0"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
|
||||
checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de"
|
||||
dependencies = [
|
||||
"curve25519-dalek",
|
||||
"curve25519-dalek 5.0.0",
|
||||
"ed25519",
|
||||
"serde",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
"signature 3.0.0",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -883,7 +915,7 @@ dependencies = [
|
||||
"generic-array",
|
||||
"group",
|
||||
"pem-rfc7468",
|
||||
"pkcs8",
|
||||
"pkcs8 0.10.2",
|
||||
"rand_core 0.6.4",
|
||||
"sec1",
|
||||
"subtle",
|
||||
@@ -968,6 +1000,12 @@ version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
|
||||
|
||||
[[package]]
|
||||
name = "fiat-crypto"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24"
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.29"
|
||||
@@ -1100,7 +1138,7 @@ checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1144,7 +1182,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"chrono",
|
||||
"futures",
|
||||
"gateway-core",
|
||||
@@ -1152,6 +1190,7 @@ dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"subtle",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1164,7 +1203,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -1188,7 +1227,6 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"serde",
|
||||
@@ -1224,7 +1262,6 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
@@ -1372,7 +1409,7 @@ version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http",
|
||||
@@ -1539,7 +1576,7 @@ version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -1737,19 +1774,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonwebtoken"
|
||||
version = "10.3.0"
|
||||
version = "11.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1"
|
||||
checksum = "881733cbc631fc9e472e24447ce32a64bedf2da498d6d8570b08edc87de71f65"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"js-sys",
|
||||
"pem",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signature",
|
||||
"signature 2.2.0",
|
||||
"simple_asn1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2016,7 +2054,7 @@ version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
@@ -2047,8 +2085,18 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der",
|
||||
"spki",
|
||||
"der 0.7.10",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2116,7 +2164,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"cookie",
|
||||
@@ -2126,7 +2174,7 @@ dependencies = [
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"hex",
|
||||
"jsonwebtoken",
|
||||
"reqwest",
|
||||
@@ -2150,7 +2198,6 @@ dependencies = [
|
||||
"url",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2159,13 +2206,14 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"jsonwebtoken",
|
||||
"p256",
|
||||
"reqwest",
|
||||
@@ -2410,7 +2458,7 @@ version = "0.12.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
@@ -2599,9 +2647,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
"generic-array",
|
||||
"pkcs8",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -2797,6 +2845,12 @@ dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.9"
|
||||
@@ -2856,7 +2910,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der",
|
||||
"der 0.7.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2878,7 +2942,7 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"chrono",
|
||||
@@ -2919,7 +2983,7 @@ dependencies = [
|
||||
"quote",
|
||||
"sqlx-core",
|
||||
"sqlx-macros-core",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2942,7 +3006,7 @@ dependencies = [
|
||||
"sqlx-mysql",
|
||||
"sqlx-postgres",
|
||||
"sqlx-sqlite",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"url",
|
||||
@@ -2982,7 +3046,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
@@ -3067,9 +3131,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.118"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3104,7 +3168,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3125,7 +3189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3239,7 +3303,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3415,7 +3479,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3692,7 +3756,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -3774,7 +3838,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3785,7 +3849,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3910,7 +3974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"deadpool",
|
||||
"futures",
|
||||
"http",
|
||||
@@ -3984,7 +4048,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4005,7 +4069,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4025,7 +4089,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4046,7 +4110,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4079,7 +4143,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -25,7 +25,7 @@ tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-th
|
||||
|
||||
[dev-dependencies]
|
||||
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
|
||||
syn = { version = "2.0.118", features = ["full"] }
|
||||
syn = { version = "3.0.3", features = ["full"] }
|
||||
tempfile = "3.27.0"
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//! docs/architecture.md 依赖 DAG 与生产源码纪律的 workspace 级机器校验。
|
||||
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// workspace 成员冻结清单;新增 crate 必须同步扩展本文件的依赖规则。
|
||||
const WORKSPACE_MEMBERS: &[&str] = &[
|
||||
"apps/gateway",
|
||||
"crates/gateway-admin",
|
||||
"crates/gateway-api",
|
||||
"crates/gateway-core",
|
||||
"crates/gateway-host",
|
||||
"crates/gateway-protocol",
|
||||
"crates/gateway-store",
|
||||
"crates/providers/openai",
|
||||
"crates/providers/xai",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn workspace_member_list_matches_the_frozen_dag_scope() {
|
||||
let manifest =
|
||||
fs::read_to_string(backend_root().join("Cargo.toml")).expect("read workspace manifest");
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
assert!(
|
||||
manifest.contains(&format!("\"{member}\"")),
|
||||
"workspace must include {member}"
|
||||
);
|
||||
}
|
||||
let member_count = manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| line.starts_with("\"apps/") || line.starts_with("\"crates/"))
|
||||
.count();
|
||||
assert_eq!(
|
||||
member_count,
|
||||
WORKSPACE_MEMBERS.len(),
|
||||
"new workspace members must extend the dependency DAG rules"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_core_depends_on_no_http_db_redis_or_provider_crate() {
|
||||
assert_no_dependency(
|
||||
"crates/gateway-core",
|
||||
&[
|
||||
"axum",
|
||||
"hyper",
|
||||
"reqwest",
|
||||
"sqlx",
|
||||
"redis",
|
||||
"gateway-host",
|
||||
"gateway-store",
|
||||
"provider-openai",
|
||||
"provider-xai",
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_protocol_has_no_workspace_dependencies() {
|
||||
for name in dependency_names("crates/gateway-protocol") {
|
||||
assert!(
|
||||
!name.starts_with("gateway-") && !name.starts_with("provider-"),
|
||||
"gateway-protocol must not depend on workspace crate `{name}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_crates_do_not_depend_on_each_other() {
|
||||
assert_no_dependency("crates/providers/openai", &["provider-xai"]);
|
||||
assert_no_dependency("crates/providers/xai", &["provider-openai"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_admin_stays_free_of_infrastructure_dependencies() {
|
||||
assert_no_dependency(
|
||||
"crates/gateway-admin",
|
||||
&["axum", "sqlx", "redis", "reqwest"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
let src = backend_root().join(member).join("src");
|
||||
let sources = super::rust_files(&src);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = src.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn backend_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束。
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
.expect("read member manifest");
|
||||
let mut in_dependencies = false;
|
||||
let mut names = Vec::new();
|
||||
for line in manifest.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
in_dependencies = line == "[dependencies]";
|
||||
continue;
|
||||
}
|
||||
if !in_dependencies || line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some((name, _)) = line.split_once('=') {
|
||||
names.push(name.trim().to_owned());
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
fn assert_no_dependency(member: &str, forbidden: &[&str]) {
|
||||
for name in dependency_names(member) {
|
||||
assert!(
|
||||
!forbidden.contains(&name.as_str()),
|
||||
"{member} must not depend on `{name}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
mod architecture;
|
||||
mod bootstrap;
|
||||
|
||||
use std::{
|
||||
@@ -21,7 +22,11 @@ fn app_tree_matches_frozen_terminal_manifest() {
|
||||
);
|
||||
assert_eq!(
|
||||
rust_files(&root.join("tests")),
|
||||
BTreeSet::from([PathBuf::from("bootstrap.rs"), PathBuf::from("main.rs")]),
|
||||
BTreeSet::from([
|
||||
PathBuf::from("architecture.rs"),
|
||||
PathBuf::from("bootstrap.rs"),
|
||||
PathBuf::from("main.rs"),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
argon2 = "0.5.3"
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.22.1"
|
||||
base64 = "0.23.0"
|
||||
chrono = "0.4.45"
|
||||
futures = "0.3.32"
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
@@ -27,6 +27,7 @@ thiserror = "2.0.18"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -29,7 +29,6 @@ pub enum AccountStatus {
|
||||
QuotaExhausted,
|
||||
Disabled,
|
||||
Banned,
|
||||
Attention,
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -184,13 +183,12 @@ pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub attention: u64,
|
||||
pub unavailable: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetAccountEnabled {
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
@@ -198,7 +196,6 @@ pub struct SetAccountEnabled {
|
||||
/// 账号批量删除命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteAccounts {
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -222,6 +219,8 @@ pub enum AccountConnectionTestEvent {
|
||||
},
|
||||
Failed {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
provider_error_type: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ impl AdminRequestContext {
|
||||
pub struct LoginCommand {
|
||||
pub username: Option<String>,
|
||||
pub password: String,
|
||||
pub source: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LoginCommand {
|
||||
@@ -48,7 +47,6 @@ impl std::fmt::Debug for LoginCommand {
|
||||
.debug_struct("LoginCommand")
|
||||
.field("username", &self.username)
|
||||
.field("password", &"[REDACTED]")
|
||||
.field("source", &self.source)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -65,8 +63,6 @@ pub struct LoginResult {
|
||||
pub enum LoginError {
|
||||
#[error("invalid administrator credentials")]
|
||||
InvalidCredentials,
|
||||
#[error("administrator login is throttled")]
|
||||
Throttled,
|
||||
#[error("administrator authentication is unavailable")]
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
@@ -141,7 +141,6 @@ impl fmt::Debug for ClientKeySecret {
|
||||
/// API 提交的 Client Key 创建命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
@@ -151,7 +150,6 @@ pub struct CreateClientKey {
|
||||
/// 管理用例生成 ID 与明文后的持久化命令。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct NewClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
@@ -175,7 +173,6 @@ impl fmt::Debug for NewClientKey {
|
||||
/// 修改 Client Key 的公开策略字段。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
@@ -186,7 +183,6 @@ pub struct UpdateClientKey {
|
||||
/// 修改 Client Key 启用状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetClientKeyEnabled {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
pub enabled: bool,
|
||||
}
|
||||
@@ -194,7 +190,6 @@ pub struct SetClientKeyEnabled {
|
||||
/// 删除 Client Key。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteClientKey {
|
||||
pub expected_config_revision: Revision,
|
||||
pub id: ClientApiKeyId,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,20 @@
|
||||
|
||||
use std::{num::NonZeroU32, str::FromStr};
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use chrono::{DateTime, TimeDelta, Timelike as _, Utc};
|
||||
|
||||
use super::{AdminModelError, PageSize};
|
||||
|
||||
/// 观测日界所用的东八区(UTC+8)固定偏移秒数。
|
||||
const CHINA_OFFSET_SECONDS: i64 = 8 * 60 * 60;
|
||||
|
||||
/// 将 UTC 时刻截断到东八区(UTC+8)当日零点,返回值仍为 UTC。
|
||||
#[must_use]
|
||||
pub fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + CHINA_OFFSET_SECONDS).rem_euclid(24 * 60 * 60);
|
||||
value - TimeDelta::seconds(elapsed) - TimeDelta::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
/// 外部观测查询的 UTC 时间范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeRange {
|
||||
@@ -718,7 +728,8 @@ pub struct OpsError {
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
pub status_code: Option<u16>,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
|
||||
@@ -101,7 +101,6 @@ pub struct CredentialDetails {
|
||||
/// Provider 正式文档批量导入命令。
|
||||
pub struct ImportCredentials {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub document: ProviderDocument,
|
||||
}
|
||||
|
||||
@@ -110,7 +109,6 @@ impl fmt::Debug for ImportCredentials {
|
||||
formatter
|
||||
.debug_struct("ImportCredentials")
|
||||
.field("context", &self.context)
|
||||
.field("expected_config_revision", &self.expected_config_revision)
|
||||
.field("document", &self.document)
|
||||
.finish()
|
||||
}
|
||||
@@ -169,7 +167,6 @@ pub struct PreparedCredentialImport {
|
||||
/// Admin 交给 Store 的导入事务命令。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialImportCommit {
|
||||
pub expected_config_revision: Revision,
|
||||
pub prepared: PreparedCredentialImport,
|
||||
}
|
||||
|
||||
@@ -254,7 +251,6 @@ pub enum AuthorizationMutationTarget {
|
||||
/// 必须完整进入 Provider opaque pending payload、并在 complete 后原样恢复的事务信封。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PendingAuthorizationMutation {
|
||||
expected_config_revision: Revision,
|
||||
provider_kind: ProviderKind,
|
||||
target: AuthorizationMutationTarget,
|
||||
owner_binding: AuthorizationOwnerBinding,
|
||||
@@ -263,24 +259,17 @@ pub struct PendingAuthorizationMutation {
|
||||
impl PendingAuthorizationMutation {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
expected_config_revision: Revision,
|
||||
provider_kind: ProviderKind,
|
||||
target: AuthorizationMutationTarget,
|
||||
owner_binding: AuthorizationOwnerBinding,
|
||||
) -> Self {
|
||||
Self {
|
||||
expected_config_revision,
|
||||
provider_kind,
|
||||
target,
|
||||
owner_binding,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn expected_config_revision(&self) -> Revision {
|
||||
self.expected_config_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
@@ -301,7 +290,6 @@ impl PendingAuthorizationMutation {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StartAuthorization {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub name: String,
|
||||
pub reauthorization: Option<ReauthorizationTarget>,
|
||||
}
|
||||
@@ -406,7 +394,6 @@ impl fmt::Debug for CompleteAuthorization {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialMutation {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_id: ProviderAccountId,
|
||||
}
|
||||
|
||||
@@ -422,7 +409,6 @@ pub struct CredentialMutationResult {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialDeletion {
|
||||
pub context: MutationContext,
|
||||
pub expected_config_revision: Revision,
|
||||
pub account_ids: Vec<ProviderAccountId>,
|
||||
}
|
||||
|
||||
@@ -520,7 +506,6 @@ impl fmt::Debug for PreparedCredentialRotation {
|
||||
/// Admin 交给 Store 的轮换或 refresh 事务命令。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CredentialRotationCommit {
|
||||
pub expected_config_revision: Revision,
|
||||
pub prepared: PreparedCredentialRotationFacts,
|
||||
}
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ pub struct RuntimeSettings {
|
||||
/// 原子替换运行设置的命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReplaceRuntimeSettings {
|
||||
pub expected_config_revision: Revision,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
|
||||
@@ -156,7 +156,7 @@ pub trait AccountStore: Send + Sync {
|
||||
) -> AdminStoreResult<()>;
|
||||
}
|
||||
|
||||
/// 管理员密码、会话、登录限流和安全审计。
|
||||
/// 管理员密码、会话和安全审计。
|
||||
#[async_trait]
|
||||
pub trait AuthStore: Send + Sync {
|
||||
async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult<Option<String>>;
|
||||
@@ -176,26 +176,10 @@ pub trait AuthStore: Send + Sync {
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>>;
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> AdminStoreResult<()>;
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()>;
|
||||
}
|
||||
|
||||
/// Client API Key 管理及其 revision CAS 写入。
|
||||
/// Client API Key 管理写入。
|
||||
#[async_trait]
|
||||
pub trait ClientKeyStore: Send + Sync {
|
||||
async fn list_client_keys(&self, query: ClientKeyListQuery) -> AdminStoreResult<ClientKeyPage>;
|
||||
@@ -281,7 +265,7 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage>;
|
||||
}
|
||||
|
||||
/// Runtime settings 与管理员 API Key 的 revision CAS 写入。
|
||||
/// Runtime settings 与管理员 API Key 写入。
|
||||
#[async_trait]
|
||||
pub trait SettingsStore: Send + Sync {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings>;
|
||||
@@ -296,14 +280,12 @@ pub trait SettingsStore: Send + Sync {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
key: AdminApiKey,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation>;
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation>;
|
||||
}
|
||||
|
||||
@@ -10,13 +10,12 @@ use gateway_core::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest},
|
||||
},
|
||||
error::GatewayErrorKind,
|
||||
routing::{ProviderKind, UpstreamModelId, snapshot::SnapshotControl},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
@@ -55,7 +54,6 @@ pub trait AccountsService: Send + Sync {
|
||||
async fn refresh(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expected_config_revision: Revision,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError>;
|
||||
|
||||
@@ -355,7 +353,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
async fn refresh(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expected_config_revision: Revision,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
@@ -368,7 +365,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
validate_prepared_rotation(&account, &prepared, "provider credential refresh")?;
|
||||
let result = commit_credential_refresh(
|
||||
self.accounts.as_ref(),
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
context,
|
||||
"provider credential refresh",
|
||||
@@ -454,12 +450,9 @@ impl AccountsService for DefaultAccountsService {
|
||||
}))
|
||||
.collect(),
|
||||
Err(error) => vec![AccountConnectionTestEvent::Failed {
|
||||
message: match error.kind() {
|
||||
GatewayErrorKind::InvalidRequest
|
||||
| GatewayErrorKind::Unsupported
|
||||
| GatewayErrorKind::ModelNotFound => error.safe_message().to_owned(),
|
||||
_ => "Provider connection test failed".to_owned(),
|
||||
},
|
||||
message: error.client_message().to_owned(),
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
account_status: status,
|
||||
}],
|
||||
}
|
||||
@@ -478,15 +471,15 @@ fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> Accoun
|
||||
crate::model::accounts::AccountAvailability::QuotaExhausted => {
|
||||
AccountStatus::QuotaExhausted
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Expired => AccountStatus::Expired,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid => AccountStatus::Expired,
|
||||
_ if account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now) =>
|
||||
{
|
||||
AccountStatus::Expired
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Ready => AccountStatus::Active,
|
||||
_ => AccountStatus::Attention,
|
||||
_ => AccountStatus::Active,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,9 +22,6 @@ use crate::{
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
const LOGIN_FAILURE_LIMIT: u32 = 5;
|
||||
const LOGIN_FAILURE_WINDOW_SECONDS: u64 = 15 * 60;
|
||||
|
||||
/// API 鉴权与管理员登录消费的统一服务。
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
@@ -39,6 +36,10 @@ pub trait AuthService: Send + Sync {
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
/// 会话有效期上限(366 天);超出的配置按上限截断,保证 TTL 构造与
|
||||
/// `Utc::now() + session_ttl` 永不越界。
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
|
||||
/// 单管理员认证策略的最终实现。
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
@@ -54,8 +55,8 @@ impl DefaultAuthService {
|
||||
store: Arc<dyn AuthStore>,
|
||||
) -> Self {
|
||||
let minutes = i64::try_from(session_ttl_minutes)
|
||||
.unwrap_or(i64::MAX)
|
||||
.max(1);
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES);
|
||||
Self {
|
||||
default_admin_user_id: default_admin_user_id.into(),
|
||||
session_ttl: Duration::minutes(minutes),
|
||||
@@ -63,19 +64,6 @@ impl DefaultAuthService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_failed_login(&self, source: &str) -> Result<LoginError, LoginError> {
|
||||
let throttled = self
|
||||
.store
|
||||
.record_login_failure(source, LOGIN_FAILURE_LIMIT, LOGIN_FAILURE_WINDOW_SECONDS)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
Ok(if throttled {
|
||||
LoginError::Throttled
|
||||
} else {
|
||||
LoginError::InvalidCredentials
|
||||
})
|
||||
}
|
||||
|
||||
fn auth_audit(&self, action: &str, occurred_at: chrono::DateTime<Utc>) -> AdminAuditEvent {
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
@@ -137,22 +125,13 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
|
||||
async fn login(&self, command: LoginCommand) -> Result<LoginResult, LoginError> {
|
||||
let source = normalized_login_source(&command.source);
|
||||
if self
|
||||
.store
|
||||
.login_source_is_throttled(source, LOGIN_FAILURE_LIMIT, LOGIN_FAILURE_WINDOW_SECONDS)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
{
|
||||
return Err(LoginError::Throttled);
|
||||
}
|
||||
if command
|
||||
.username
|
||||
.as_deref()
|
||||
.unwrap_or(&self.default_admin_user_id)
|
||||
!= self.default_admin_user_id
|
||||
{
|
||||
return Err(self.record_failed_login(source).await?);
|
||||
return Err(LoginError::InvalidCredentials);
|
||||
}
|
||||
let hash = self
|
||||
.store
|
||||
@@ -161,13 +140,9 @@ impl AuthService for DefaultAuthService {
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
.ok_or(LoginError::InvalidCredentials)?;
|
||||
if !verify_admin_password(&command.password, &hash).map_err(|_| LoginError::Unavailable)? {
|
||||
return Err(self.record_failed_login(source).await?);
|
||||
return Err(LoginError::InvalidCredentials);
|
||||
}
|
||||
|
||||
self.store
|
||||
.clear_login_failures(source)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
let session_id = random_session_token();
|
||||
let expires_at = Utc::now() + self.session_ttl;
|
||||
self.store
|
||||
@@ -246,8 +221,3 @@ fn valid_admin_api_key_shape(value: &str) -> bool {
|
||||
&& value.starts_with("admin-")
|
||||
&& value[6..].bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
fn normalized_login_source(source: &str) -> &str {
|
||||
let source = source.trim();
|
||||
if source.is_empty() { "unknown" } else { source }
|
||||
}
|
||||
|
||||
@@ -94,7 +94,6 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
.store
|
||||
.create_client_key(
|
||||
NewClientKey {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
id,
|
||||
name: command.name,
|
||||
label: command.label,
|
||||
|
||||
@@ -108,7 +108,6 @@ async fn pending_authorization(
|
||||
},
|
||||
};
|
||||
Ok(PendingAuthorizationMutation::new(
|
||||
command.expected_config_revision,
|
||||
provider_kind.clone(),
|
||||
target,
|
||||
crate::model::provider_credentials::AuthorizationOwnerBinding::from_context(
|
||||
@@ -215,20 +214,13 @@ async fn commit_authorization(
|
||||
|
||||
async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
expected_config_revision: crate::model::Revision,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
expected_config_revision,
|
||||
prepared: facts,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.commit_credential_rotation(CredentialRotationCommit { prepared: facts }, context)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -244,20 +236,13 @@ async fn commit_credential_rotation(
|
||||
|
||||
async fn commit_credential_refresh(
|
||||
accounts: &dyn AccountStore,
|
||||
expected_config_revision: crate::model::Revision,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
expected_config_revision,
|
||||
prepared: facts,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.commit_credential_refresh(CredentialRotationCommit { prepared: facts }, context)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
@@ -289,7 +274,6 @@ async fn set_credential_enabled(
|
||||
let revision = accounts
|
||||
.set_account_enabled(
|
||||
SetAccountEnabled {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
enabled,
|
||||
},
|
||||
@@ -320,7 +304,6 @@ async fn delete_credentials(
|
||||
let revision = accounts
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
expected_config_revision: command.expected_config_revision,
|
||||
account_ids: account_ids
|
||||
.iter()
|
||||
.map(|account_id| account_id.as_str().to_owned())
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::{
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost,
|
||||
UsageInsightsCostPoint, UsageInsightsHealth, UsageInsightsHealthPoint,
|
||||
UsageInsightsPerformance, UsageInsightsPerformancePoint, UsageOverview, UsagePage,
|
||||
UsageQuery, UsageSummary,
|
||||
UsageQuery, UsageSummary, china_day_start,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
@@ -38,7 +38,6 @@ const HEALTH_TIMELINE_SLOTS: i64 = 24 * 4;
|
||||
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
|
||||
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
|
||||
const CHINA_OFFSET_SECONDS: i64 = 8 * 60 * 60;
|
||||
|
||||
/// API 消费的观测控制面服务。
|
||||
#[async_trait]
|
||||
@@ -105,7 +104,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
self.store.dashboard_runtime_slots(range.end),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
self.enrich_billing(&mut observation.recent_requests)?;
|
||||
self.enrich_billing(&mut observation.recent_requests);
|
||||
self.enrich_dashboard_quotas(&mut observation.account_usage)
|
||||
.await;
|
||||
let today_start = china_day_start(observation.range.end);
|
||||
@@ -177,7 +176,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.list_usage_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage records"))?;
|
||||
self.enrich_billing(&mut page.items)?;
|
||||
self.enrich_billing(&mut page.items);
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -190,7 +189,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.usage_record_detail(request_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage record"))?;
|
||||
self.enrich_billing(std::slice::from_mut(&mut detail.request))?;
|
||||
self.enrich_billing(std::slice::from_mut(&mut detail.request));
|
||||
Ok(detail)
|
||||
}
|
||||
|
||||
@@ -492,10 +491,9 @@ impl DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
fn enrich_billing(
|
||||
&self,
|
||||
records: &mut [crate::model::observability::UsageRecord],
|
||||
) -> Result<(), AdminError> {
|
||||
/// 逐条尽力把 calculated 总额升级为完整分解;单条脏数据(非法 Provider kind
|
||||
/// 或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
|
||||
for record in records {
|
||||
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
|
||||
continue;
|
||||
@@ -509,8 +507,9 @@ impl DefaultObservabilityService {
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
let provider_kind = ProviderKind::new(provider.to_owned())
|
||||
.map_err(|_| AdminError::internal("Stored Provider kind is invalid"))?;
|
||||
let Ok(provider_kind) = ProviderKind::new(provider.to_owned()) else {
|
||||
continue;
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: upstream_model_id.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
@@ -519,15 +518,10 @@ impl DefaultObservabilityService {
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Some(breakdown) = self
|
||||
.providers
|
||||
.calculated_billing(&provider_kind, &input)
|
||||
.map_err(|error| map_provider_error(error, "usage billing"))?
|
||||
{
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
record.billing = Some(UsageBilling::Calculated(Box::new(breakdown)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -833,11 +827,6 @@ fn service_failure_count(metrics: &RequestMetrics) -> u64 {
|
||||
.saturating_sub(metrics.caller_error_count)
|
||||
}
|
||||
|
||||
fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + CHINA_OFFSET_SECONDS).rem_euclid(24 * 60 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
fn quarter_hour_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = value
|
||||
.timestamp()
|
||||
|
||||
@@ -112,7 +112,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
command: ImportCredentials,
|
||||
) -> Result<CredentialImportResult, AdminError> {
|
||||
let context = command.context;
|
||||
let expected_config_revision = command.expected_config_revision;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_import(PrepareCredentialImport {
|
||||
@@ -127,13 +126,7 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
)?;
|
||||
let result = self
|
||||
.accounts
|
||||
.commit_credential_import(
|
||||
CredentialImportCommit {
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "OpenAI credential import"))?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
@@ -200,7 +193,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
command: RotateCredential,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let context = command.mutation.context;
|
||||
let expected_config_revision = command.mutation.expected_config_revision;
|
||||
let account_id = command.mutation.account_id;
|
||||
let details = required_credential(
|
||||
self.accounts.as_ref(),
|
||||
@@ -227,7 +219,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
validate_prepared_rotation(&account, &prepared, "OpenAI credential rotation")?;
|
||||
let result = commit_credential_rotation(
|
||||
self.accounts.as_ref(),
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
&context,
|
||||
"OpenAI credential rotation",
|
||||
|
||||
@@ -86,13 +86,12 @@ impl SettingsService for DefaultSettingsService {
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<RegeneratedAdminApiKey, AdminError> {
|
||||
let settings = self.load().await?;
|
||||
let mut bytes = [0_u8; 32];
|
||||
OsRng.fill_bytes(&mut bytes);
|
||||
let key = AdminApiKey::new(format!("admin-{}", hex::encode(bytes)));
|
||||
let mutation = self
|
||||
.store
|
||||
.replace_admin_api_key(settings.config_revision, key.clone(), context)
|
||||
.replace_admin_api_key(key.clone(), context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator API key"))?;
|
||||
publish_committed(self.snapshot.as_ref(), mutation.config_revision).await?;
|
||||
@@ -103,10 +102,9 @@ impl SettingsService for DefaultSettingsService {
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<AdminApiKeyMutation, AdminError> {
|
||||
let settings = self.load().await?;
|
||||
let mutation = self
|
||||
.store
|
||||
.delete_admin_api_key(settings.config_revision, context)
|
||||
.delete_admin_api_key(context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator API key"))?;
|
||||
publish_committed(self.snapshot.as_ref(), mutation.config_revision).await?;
|
||||
|
||||
@@ -89,7 +89,6 @@ impl XaiService for DefaultXaiService {
|
||||
command: ImportCredentials,
|
||||
) -> Result<CredentialImportResult, AdminError> {
|
||||
let context = command.context;
|
||||
let expected_config_revision = command.expected_config_revision;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_import(PrepareCredentialImport {
|
||||
@@ -104,13 +103,7 @@ impl XaiService for DefaultXaiService {
|
||||
)?;
|
||||
let result = self
|
||||
.accounts
|
||||
.commit_credential_import(
|
||||
CredentialImportCommit {
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "xAI credential import"))?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
|
||||
@@ -87,7 +87,6 @@ fn login_command_debug_should_redact_password() {
|
||||
let command = LoginCommand {
|
||||
username: None,
|
||||
password: "secret-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
};
|
||||
assert!(!format!("{command:?}").contains("secret-password"));
|
||||
}
|
||||
|
||||
@@ -2,13 +2,14 @@ use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
operation::{GenerateRequest, Operation, ProtocolPayload},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
|
||||
@@ -17,9 +18,9 @@ use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountListQuery, AccountPage, AccountRecord, AccountSummary,
|
||||
AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts,
|
||||
SetAccountEnabled,
|
||||
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
|
||||
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -48,6 +49,7 @@ use gateway_admin::{
|
||||
},
|
||||
},
|
||||
};
|
||||
use serde_json::{Map, json};
|
||||
|
||||
pub(super) type EventLog = Arc<Mutex<Vec<&'static str>>>;
|
||||
|
||||
@@ -159,10 +161,22 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
_: &gateway_core::routing::UpstreamModelId,
|
||||
_: &str,
|
||||
model: &gateway_core::routing::UpstreamModelId,
|
||||
input: &str,
|
||||
) -> Result<gateway_core::operation::Operation, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!(model.as_str())),
|
||||
("input".to_owned(), json!(input)),
|
||||
("stream".to_owned(), json!(true)),
|
||||
("store".to_owned(), json!(false)),
|
||||
]),
|
||||
)
|
||||
.map_err(|_| ProviderAdminError::new(ProviderAdminErrorKind::Invalid))?;
|
||||
Ok(Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
payload,
|
||||
)))
|
||||
}
|
||||
|
||||
fn dashboard_wire_profile(
|
||||
@@ -328,18 +342,20 @@ pub(super) struct FakeAccountStore {
|
||||
account: AccountRecord,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
authorization_revision: Mutex<Option<Revision>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
pub(super) fn new(kind: &str, events: EventLog) -> Arc<Self> {
|
||||
Self::with_account(account_record(kind), events)
|
||||
}
|
||||
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account: account_record(kind),
|
||||
account,
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
authorization_revision: Mutex::new(None),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
@@ -352,13 +368,6 @@ impl FakeAccountStore {
|
||||
self.audit_requests.lock().expect("audit requests").clone()
|
||||
}
|
||||
|
||||
pub(super) fn authorization_revision(&self) -> Option<Revision> {
|
||||
*self
|
||||
.authorization_revision
|
||||
.lock()
|
||||
.expect("authorization revision")
|
||||
}
|
||||
|
||||
pub(super) fn set_quota_window_usage(&self, usage: Vec<AccountUsageWindowResult>) {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
@@ -395,7 +404,7 @@ impl AccountStore for FakeAccountStore {
|
||||
total: 1,
|
||||
active: 1,
|
||||
quota_exhausted: 0,
|
||||
attention: 0,
|
||||
unavailable: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -505,10 +514,6 @@ impl AccountStore for FakeAccountStore {
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.record("store.commit_authorization");
|
||||
self.record_context(context);
|
||||
*self
|
||||
.authorization_revision
|
||||
.lock()
|
||||
.expect("authorization revision") = Some(command.pending.expected_config_revision());
|
||||
self.require_commit()?;
|
||||
let (account_id, credential_revision) = match command.credential {
|
||||
AuthorizationCredentialCommit::Create(credential) => (credential.account_id, None),
|
||||
@@ -613,7 +618,6 @@ impl SettingsStore for StaticSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -622,7 +626,6 @@ impl SettingsStore for StaticSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
@@ -686,6 +689,39 @@ fn provider_registry_should_reject_duplicate_kind() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_probe_unavailable_account() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
&& error_type == "invalid_request_error"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -724,7 +760,6 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-request"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
@@ -837,7 +872,6 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-provider-error"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
@@ -850,7 +884,7 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_store_failure_should_drop_guard_after_cas_attempt() {
|
||||
async fn accounts_refresh_store_failure_should_drop_guard_after_commit_attempt() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
@@ -861,11 +895,10 @@ async fn accounts_refresh_store_failure_should_drop_guard_after_cas_attempt() {
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-store-error"),
|
||||
revision(1),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
.expect_err("Store CAS must fail");
|
||||
.expect_err("Store commit must fail");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
@@ -994,7 +1027,7 @@ fn prepared_create_with_id(
|
||||
|
||||
fn rotation_result(command: CredentialRotationCommit) -> CredentialMutationResult {
|
||||
CredentialMutationResult {
|
||||
config_revision: revision(command.expected_config_revision.get() + 1),
|
||||
config_revision: revision(2),
|
||||
account_id: command.prepared.account_id,
|
||||
credential_revision: Some(revision(
|
||||
command.prepared.expected_credential_revision.get() + 1,
|
||||
@@ -1005,12 +1038,20 @@ fn rotation_result(command: CredentialRotationCommit) -> CredentialMutationResul
|
||||
async fn accounts_service(
|
||||
provider: Arc<FakeProviderAdmin>,
|
||||
store: Arc<FakeAccountStore>,
|
||||
) -> AdminServices {
|
||||
accounts_service_with_probe(provider, store, Arc::new(SuccessfulAccountProbe)).await
|
||||
}
|
||||
|
||||
async fn accounts_service_with_probe(
|
||||
provider: Arc<FakeProviderAdmin>,
|
||||
store: Arc<FakeAccountStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> AdminServices {
|
||||
super::AdminHarness::new()
|
||||
.accounts(store)
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(provider)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.probe(probe)
|
||||
.build()
|
||||
.await
|
||||
}
|
||||
@@ -1030,6 +1071,29 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingAccountProbe;
|
||||
|
||||
impl AccountProbe for FailingAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
Box::pin(async {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"included usage exhausted",
|
||||
Some("usage_exhausted".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider error");
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
)
|
||||
.with_client_visible_upstream_error(detail))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{collections::BTreeMap, sync::Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
@@ -15,7 +16,6 @@ struct MemoryAuthStore {
|
||||
password_hash: Mutex<Option<String>>,
|
||||
sessions: Mutex<BTreeMap<String, AdminSession>>,
|
||||
audits: Mutex<Vec<AdminAuditEvent>>,
|
||||
failures: Mutex<u32>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -66,31 +66,6 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(self.sessions.lock().expect("sessions").remove(session_id))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
_: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(*self.failures.lock().expect("failures") >= failure_limit)
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
_: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut failures = self.failures.lock().expect("failures");
|
||||
*failures += 1;
|
||||
Ok(*failures >= failure_limit)
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, _: &str) -> AdminStoreResult<()> {
|
||||
*self.failures.lock().expect("failures") = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
self.audits.lock().expect("audits").push(event);
|
||||
Ok(())
|
||||
@@ -107,7 +82,6 @@ async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "strong-test-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("login");
|
||||
@@ -142,9 +116,38 @@ async fn repeated_default_initialization_should_not_replace_password() {
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "first-strong-password".to_owned(),
|
||||
source: "127.0.0.1".to_owned(),
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn login_with_huge_session_ttl_should_clamp_expiry_instead_of_panicking() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.auth(store.clone())
|
||||
.session_ttl_minutes(i64::MAX as u64)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let before = Utc::now();
|
||||
let result = services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
username: Some("admin".to_owned()),
|
||||
password: "strong-test-password".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("login with huge session TTL");
|
||||
|
||||
assert!(result.expires_at > before);
|
||||
assert!(result.expires_at <= before + TimeDelta::days(366) + TimeDelta::minutes(1));
|
||||
assert!(
|
||||
services
|
||||
.auth()
|
||||
.validate_session(Some(&result.session_id))
|
||||
.await
|
||||
.expect("validate")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -69,6 +69,7 @@ use gateway_core::{
|
||||
|
||||
pub(super) struct AdminHarness {
|
||||
default_password: String,
|
||||
session_ttl_minutes: u64,
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
@@ -84,6 +85,7 @@ impl AdminHarness {
|
||||
let unavailable = Arc::new(UnavailableStore);
|
||||
Self {
|
||||
default_password: "strong-test-password".to_owned(),
|
||||
session_ttl_minutes: 60,
|
||||
accounts: unavailable.clone(),
|
||||
auth: Arc::new(BootstrapAuthStore::default()),
|
||||
client_keys: unavailable.clone(),
|
||||
@@ -103,6 +105,11 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn session_ttl_minutes(mut self, minutes: u64) -> Self {
|
||||
self.session_ttl_minutes = minutes;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn accounts(mut self, store: Arc<dyn AccountStore>) -> Self {
|
||||
self.accounts = store;
|
||||
self
|
||||
@@ -148,7 +155,7 @@ impl AdminHarness {
|
||||
pub(super) async fn build(self) -> AdminServices {
|
||||
gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_ttl_minutes: 60,
|
||||
session_ttl_minutes: self.session_ttl_minutes,
|
||||
default_username: "admin".to_owned(),
|
||||
default_password: InitialAdminPassword::new(self.default_password),
|
||||
},
|
||||
@@ -210,18 +217,6 @@ impl AuthStore for BootstrapAuthStore {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(&self, _: &str, _: u32, _: u64) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn record_login_failure(&self, _: &str, _: u32, _: u64) -> AdminStoreResult<bool> {
|
||||
Err(unavailable("login failure"))
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, _: &str) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, _: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -454,7 +449,6 @@ impl SettingsStore for UnavailableStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -463,7 +457,6 @@ impl SettingsStore for UnavailableStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(unavailable("admin API key"))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::{
|
||||
num::NonZeroU32,
|
||||
str::FromStr as _,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
@@ -9,13 +10,14 @@ use chrono::{DateTime, Duration, Timelike as _, Utc};
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
MutationContext, PageSize, Revision,
|
||||
observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, CostCoverage, CurrencyCost, DashboardObservation,
|
||||
DashboardRuntimeSlots, DiagnosticDimension, DiagnosticObservation, Granularity,
|
||||
HealthStatus, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, RequestMetrics,
|
||||
TimeRange, TrendKind, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
HealthStatus, OpsErrorPage, OpsErrorQuery, PageNumber, RequestMetricPoint,
|
||||
RequestMetrics, RequestOutcome, TimeRange, TrendKind, UsageBilling,
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageOverview, UsagePage,
|
||||
UsageQuery, UsageRecord, china_day_start,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -318,12 +320,46 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
assert_eq!(diagnostics.items[1].request_share, 0.25);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let invalid_kind = "x".repeat(65);
|
||||
store.replace_usage_records(vec![
|
||||
calculated_total_record("request_invalid_kind", Some(&invalid_kind), now),
|
||||
calculated_total_record("request_unregistered_kind", Some("anthropic"), now),
|
||||
calculated_total_record("request_enriched", Some("openai"), now),
|
||||
]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("one bad record must not fail the whole usage list");
|
||||
|
||||
assert_eq!(page.items.len(), 3);
|
||||
assert!(
|
||||
matches!(page.items[0].billing, Some(UsageBilling::Total { .. })),
|
||||
"invalid Provider kind keeps the stored total"
|
||||
);
|
||||
assert!(
|
||||
matches!(page.items[1].billing, Some(UsageBilling::Total { .. })),
|
||||
"unregistered Provider kind keeps the stored total"
|
||||
);
|
||||
assert!(
|
||||
matches!(page.items[2].billing, Some(UsageBilling::Calculated(_))),
|
||||
"healthy record is still enriched"
|
||||
);
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
|
||||
diagnostics: Mutex<Vec<DiagnosticObservation>>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
usage_records: Mutex<Vec<UsageRecord>>,
|
||||
}
|
||||
|
||||
impl FixtureObservabilityStore {
|
||||
@@ -339,6 +375,7 @@ impl FixtureObservabilityStore {
|
||||
calculated_billing_facts: Mutex::new(Vec::new()),
|
||||
diagnostics: Mutex::new(Vec::new()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
usage_records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +401,10 @@ impl FixtureObservabilityStore {
|
||||
fn replace_runtime_slots(&self, runtime_slots: Option<DashboardRuntimeSlots>) {
|
||||
*self.runtime_slots.lock().expect("runtime slots") = runtime_slots;
|
||||
}
|
||||
|
||||
fn replace_usage_records(&self, records: Vec<UsageRecord>) {
|
||||
*self.usage_records.lock().expect("usage records") = records;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -412,7 +453,13 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
Err(super::unavailable("usage records"))
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
||||
Ok(UsagePage {
|
||||
items,
|
||||
total,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
@@ -471,7 +518,6 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -480,7 +526,6 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(super::unavailable("admin API key"))
|
||||
@@ -511,6 +556,96 @@ fn observation_range(end: DateTime<Utc>) -> TimeRange {
|
||||
TimeRange::new(end - Duration::hours(24), end).expect("observation range")
|
||||
}
|
||||
|
||||
fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
UsageQuery {
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
cursor: None,
|
||||
page: PageNumber::new(NonZeroU32::new(1).expect("page number")),
|
||||
page_size: PageSize::new(50).expect("page size"),
|
||||
}
|
||||
}
|
||||
|
||||
fn calculated_total_record(
|
||||
id: &str,
|
||||
provider_kind: Option<&str>,
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageRecord {
|
||||
UsageRecord {
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_billing".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "gpt-5.5".to_owned(),
|
||||
provider_kind: provider_kind.map(str::to_owned),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
provider_account_email: None,
|
||||
provider_account_authentication_kind: None,
|
||||
upstream_model_id: Some("gpt-5.5".to_owned()),
|
||||
upstream_transport: None,
|
||||
http_version: None,
|
||||
websocket_pool: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
cost_source: "calculated".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Total {
|
||||
source: "calculated".to_owned(),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
.expect("billing total"),
|
||||
},
|
||||
}),
|
||||
transport_decision_wait_ms: None,
|
||||
connect_ms: None,
|
||||
headers_ms: None,
|
||||
first_event_ms: None,
|
||||
first_reasoning_ms: None,
|
||||
first_text_ms: None,
|
||||
first_token_ms: None,
|
||||
provider_processing_ms: None,
|
||||
latency_ms: Some(100),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
image_generation_succeeded: None,
|
||||
started_at: now,
|
||||
deadline_at: now,
|
||||
completed_at: Some(now),
|
||||
}
|
||||
}
|
||||
|
||||
fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) -> RequestMetricPoint {
|
||||
RequestMetricPoint {
|
||||
bucket_start,
|
||||
@@ -535,11 +670,6 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn china_day_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = (value.timestamp() + 8 * 60 * 60).rem_euclid(24 * 60 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
}
|
||||
|
||||
fn quarter_hour_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let elapsed = value.timestamp().rem_euclid(15 * 60);
|
||||
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
||||
|
||||
@@ -114,7 +114,6 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -151,7 +150,6 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-batch"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -193,7 +191,6 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-quota-failure"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -218,7 +215,6 @@ async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
.openai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-openai-error"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -237,7 +233,6 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai"),
|
||||
expected_config_revision: revision(7),
|
||||
name: "reauthorize".to_owned(),
|
||||
reauthorization: Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
@@ -248,7 +243,6 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.expect("start reauthorization");
|
||||
|
||||
let pending = provider.pending().expect("pending envelope");
|
||||
assert_eq!(pending.expected_config_revision(), revision(7));
|
||||
assert_eq!(pending.provider_kind().as_str(), "openai");
|
||||
assert_eq!(
|
||||
pending.owner_binding().started_request_id(),
|
||||
@@ -284,7 +278,6 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
"guard.finish",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.authorization_revision(), Some(revision(7)));
|
||||
assert_eq!(store.audit_requests(), ["oauth-complete-openai"]);
|
||||
}
|
||||
|
||||
@@ -299,7 +292,6 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
}
|
||||
}
|
||||
@@ -307,7 +299,6 @@ fn mutation(account_id: &str) -> CredentialMutation {
|
||||
fn deletion(account_id: &str) -> CredentialDeletion {
|
||||
CredentialDeletion {
|
||||
context: context("request-openai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_ids: vec![ProviderAccountId::new(account_id).expect("account ID")],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use async_trait::async_trait;
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
AdminErrorKind, MutationContext, Revision,
|
||||
AdminErrorKind, MutationContext,
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
RuntimeSettings,
|
||||
@@ -33,7 +33,6 @@ impl SettingsStore for UnusedSettingsStore {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -42,7 +41,6 @@ impl SettingsStore for UnusedSettingsStore {
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(unused())
|
||||
@@ -63,7 +61,6 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
request_id: "request-settings".to_owned(),
|
||||
},
|
||||
ReplaceRuntimeSettings {
|
||||
expected_config_revision: Revision::new(1).expect("revision"),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
|
||||
@@ -8,9 +8,7 @@ use gateway_admin::{
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
|
||||
use super::accounts::{
|
||||
FakeAccountStore, FakeProviderAdmin, context, document, events, recorded, revision,
|
||||
};
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, context, document, events, recorded};
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
@@ -47,7 +45,6 @@ async fn xai_import_should_prepare_before_atomic_store_commit() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai"),
|
||||
expected_config_revision: revision(3),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -84,7 +81,6 @@ async fn xai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai-batch"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -119,7 +115,6 @@ async fn xai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.xai()
|
||||
.import_document(ImportCredentials {
|
||||
context: context("import-xai-quota-failure"),
|
||||
expected_config_revision: revision(1),
|
||||
document: document(),
|
||||
})
|
||||
.await
|
||||
@@ -140,7 +135,6 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-xai"),
|
||||
expected_config_revision: revision(1),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.8.9", default-features = false, features = ["json", "query", "ws"] }
|
||||
base64 = "0.22.1"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
futures = "0.3.32"
|
||||
|
||||
@@ -161,7 +161,6 @@ fn parse_sort_direction(value: &str) -> Option<SortDirection> {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountPageData {
|
||||
pub config_revision: u64,
|
||||
pub items: Vec<AccountView>,
|
||||
pub page: PageMeta,
|
||||
pub summary: AccountSummaryView,
|
||||
@@ -174,7 +173,7 @@ pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub attention: u64,
|
||||
pub unavailable: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -391,31 +390,21 @@ impl AccountActionRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 手工 OAuth 刷新会变更持久 credential,因此必须携带配置 CAS revision。
|
||||
/// 手工 OAuth 刷新会变更持久 credential。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountRefreshRequest {
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountRefreshRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
if self.expected_config_revision == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
Ok(())
|
||||
require_account_id(&self.account_id, "accountId")
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<(Revision, ProviderAccountId), WireValidationError> {
|
||||
fn into_command(self) -> Result<ProviderAccountId, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok((
|
||||
Revision::new(self.expected_config_revision)
|
||||
.map_err(|_| WireValidationError::new("expectedConfigRevision"))?,
|
||||
ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
))
|
||||
ProviderAccountId::new(self.account_id).map_err(|_| WireValidationError::new("accountId"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,7 +449,6 @@ pub struct AccountModelsData {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountRefreshData {
|
||||
pub config_revision: u64,
|
||||
pub account: AccountView,
|
||||
}
|
||||
|
||||
@@ -543,10 +531,14 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code,
|
||||
provider_error_type,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
"providerErrorCode": provider_error_code,
|
||||
"providerErrorType": provider_error_type,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
}),
|
||||
};
|
||||
@@ -575,14 +567,12 @@ impl AccountProvider {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountImportRequest {
|
||||
pub provider: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub data: Value,
|
||||
}
|
||||
|
||||
impl AccountImportRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
if !self.data.is_object()
|
||||
|| serde_json::to_vec(&self.data)
|
||||
.map_or(true, |encoded| encoded.len() > MAX_IMPORT_DATA_BYTES)
|
||||
@@ -602,10 +592,6 @@ impl AccountImportRequest {
|
||||
provider,
|
||||
ImportCredentials {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
document: provider_document(self.data, "data")?,
|
||||
},
|
||||
))
|
||||
@@ -616,7 +602,6 @@ impl AccountImportRequest {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct StartAccountAuthorizationRequest {
|
||||
pub provider: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub name: String,
|
||||
pub account_id: Option<String>,
|
||||
pub expected_credential_revision: Option<u64>,
|
||||
@@ -625,7 +610,6 @@ pub struct StartAccountAuthorizationRequest {
|
||||
impl StartAccountAuthorizationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
require_text(&self.name, MAX_NAME_BYTES, "name")?;
|
||||
match (
|
||||
self.account_id.as_deref(),
|
||||
@@ -659,10 +643,6 @@ impl StartAccountAuthorizationRequest {
|
||||
provider,
|
||||
StartAuthorization {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
name: self.name,
|
||||
reauthorization,
|
||||
},
|
||||
@@ -717,14 +697,12 @@ impl CompleteAccountAuthorizationRequest {
|
||||
pub struct AccountMutationRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountMutationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")
|
||||
require_account_id(&self.account_id, "accountId")
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -737,10 +715,6 @@ impl AccountMutationRequest {
|
||||
provider,
|
||||
CredentialMutation {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
@@ -753,7 +727,6 @@ impl AccountMutationRequest {
|
||||
pub struct AccountDeletionRequest {
|
||||
pub provider: String,
|
||||
pub account_ids: Vec<String>,
|
||||
pub expected_config_revision: u64,
|
||||
}
|
||||
|
||||
impl AccountDeletionRequest {
|
||||
@@ -769,7 +742,7 @@ impl AccountDeletionRequest {
|
||||
return Err(WireValidationError::new("accountIds"));
|
||||
}
|
||||
}
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -782,10 +755,6 @@ impl AccountDeletionRequest {
|
||||
provider,
|
||||
CredentialDeletion {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_ids: self
|
||||
.account_ids
|
||||
.into_iter()
|
||||
@@ -802,7 +771,6 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_config_revision: u64,
|
||||
pub expected_credential_revision: u64,
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
@@ -814,7 +782,6 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(self.expected_config_revision, "expectedConfigRevision")?;
|
||||
require_positive(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
@@ -836,10 +803,6 @@ impl RotateAccountRequest {
|
||||
Ok(RotateCredential {
|
||||
mutation: CredentialMutation {
|
||||
context,
|
||||
expected_config_revision: revision(
|
||||
self.expected_config_revision,
|
||||
"expectedConfigRevision",
|
||||
)?,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
@@ -855,7 +818,6 @@ impl RotateAccountRequest {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountImportData {
|
||||
pub config_revision: u64,
|
||||
pub imported_count: usize,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
@@ -868,7 +830,6 @@ impl From<CredentialImportResult> for AccountImportData {
|
||||
.map(|account_id| account_id.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
imported_count: account_ids.len(),
|
||||
account_ids,
|
||||
}
|
||||
@@ -896,7 +857,6 @@ impl From<AuthorizationStarted> for AccountAuthorizationData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountMutationData {
|
||||
pub config_revision: u64,
|
||||
pub account_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_revision: Option<u64>,
|
||||
@@ -905,7 +865,6 @@ pub struct AccountMutationData {
|
||||
impl From<CredentialMutationResult> for AccountMutationData {
|
||||
fn from(result: CredentialMutationResult) -> Self {
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
account_id: result.account_id.to_string(),
|
||||
credential_revision: result.credential_revision.map(Revision::get),
|
||||
}
|
||||
@@ -915,7 +874,6 @@ impl From<CredentialMutationResult> for AccountMutationData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountDeletionData {
|
||||
pub config_revision: u64,
|
||||
pub deleted_count: usize,
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
@@ -928,7 +886,6 @@ impl From<CredentialDeletionResult> for AccountDeletionData {
|
||||
.map(|account_id| account_id.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
Self {
|
||||
config_revision: result.config_revision.get(),
|
||||
deleted_count: account_ids.len(),
|
||||
account_ids,
|
||||
}
|
||||
@@ -1260,15 +1217,11 @@ async fn refresh_account<S>(
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (expected_config_revision, account_id) = request.into_command().map_err(map_wire_error)?;
|
||||
let account_id = request.into_command().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.refresh(
|
||||
&auth.context().mutation_context(),
|
||||
expected_config_revision,
|
||||
account_id,
|
||||
)
|
||||
.refresh(&auth.context().mutation_context(), account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_refresh_data(result, Utc::now());
|
||||
@@ -1390,7 +1343,6 @@ fn account_page_data(
|
||||
u32::try_from(result.total.div_ceil(u64::from(page_size))).unwrap_or(u32::MAX)
|
||||
};
|
||||
AccountPageData {
|
||||
config_revision: result.config_revision.get(),
|
||||
items: result
|
||||
.items
|
||||
.into_iter()
|
||||
@@ -1401,14 +1353,13 @@ fn account_page_data(
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
attention: result.summary.attention,
|
||||
unavailable: result.summary.unavailable,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn account_refresh_data(result: AccountRefreshResult, now: DateTime<Utc>) -> AccountRefreshData {
|
||||
AccountRefreshData {
|
||||
config_revision: result.config_revision.get(),
|
||||
account: account_view(result.account, now),
|
||||
}
|
||||
}
|
||||
@@ -1798,7 +1749,6 @@ fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
DomainAccountStatus::Attention => "attention",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::fmt;
|
||||
|
||||
use axum::{
|
||||
Extension, Json, Router,
|
||||
Json, Router,
|
||||
extract::{FromRequestParts, State},
|
||||
http::{HeaderMap, HeaderValue, StatusCode, header::SET_COOKIE, request::Parts},
|
||||
response::{IntoResponse, Response},
|
||||
@@ -14,6 +14,7 @@ use gateway_admin::{
|
||||
model::auth::{AdminPrincipal, AdminRequestContext, LoginCommand, LoginError},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tower_http::request_id::RequestId;
|
||||
|
||||
use super::{AdminEnvelope, AdminError, AdminResponse};
|
||||
|
||||
@@ -51,22 +52,32 @@ where
|
||||
|
||||
async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
|
||||
let principal = require_admin_auth(state, &parts.headers).await?;
|
||||
let request_id = parts
|
||||
.headers
|
||||
.get(REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
let request_id = admin_request_id(parts)
|
||||
.ok_or_else(|| AdminError::internal("Missing admin request context"))?;
|
||||
Ok(Self {
|
||||
context: AdminRequestContext {
|
||||
principal,
|
||||
request_id: request_id.to_owned(),
|
||||
request_id,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// request-id 层按配置的 header 名注入,同时写入与名字无关的扩展;
|
||||
/// 优先读扩展,使自定义 header 名不会让管理请求失去请求上下文。
|
||||
/// header 回退覆盖未装配该层的调用方。
|
||||
fn admin_request_id(parts: &Parts) -> Option<String> {
|
||||
parts
|
||||
.extensions
|
||||
.get::<RequestId>()
|
||||
.map(RequestId::header_value)
|
||||
.or_else(|| parts.headers.get(REQUEST_ID_HEADER))
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub async fn require_admin_session<S>(state: &S, headers: &HeaderMap) -> Result<String, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
@@ -185,22 +196,6 @@ impl Default for AdminLogoutData {
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求上下文解析出的客户端登录来源。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AdminLoginSource(String);
|
||||
|
||||
impl AdminLoginSource {
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造固定 GET/POST 管理员认证路由。
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
@@ -214,24 +209,16 @@ where
|
||||
|
||||
async fn login<S>(
|
||||
State(state): State<S>,
|
||||
source: Option<Extension<AdminLoginSource>>,
|
||||
Json(payload): Json<AdminLoginRequest>,
|
||||
) -> Result<Response, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let source = source
|
||||
.as_ref()
|
||||
.map_or("unknown", |Extension(source)| source.as_str());
|
||||
let (username, password) = payload.into_parts();
|
||||
let session = state
|
||||
.admin_services()
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
username,
|
||||
password,
|
||||
source: source.to_owned(),
|
||||
})
|
||||
.login(LoginCommand { username, password })
|
||||
.await
|
||||
.map_err(map_login_error)?;
|
||||
let mut response = AdminResponse::new(
|
||||
@@ -300,7 +287,6 @@ fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
|
||||
fn map_login_error(error: LoginError) -> AdminError {
|
||||
match error {
|
||||
LoginError::InvalidCredentials => AdminError::invalid_admin_credentials(),
|
||||
LoginError::Throttled => AdminError::too_many_login_attempts(),
|
||||
LoginError::Unavailable => AdminError::internal("Failed to create admin session"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,14 +4,11 @@ use std::fmt;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
client_keys::{
|
||||
ClientKeyCursor, ClientKeyCursorValue as DomainCursorValue, ClientKeyListQuery,
|
||||
ClientKeyMutation, ClientKeyPage, ClientKeyPageSize, ClientKeyRecord, ClientKeySecret,
|
||||
ClientKeySort as DomainSort, ClientKeySortField as DomainSortField, CreateClientKey,
|
||||
CreatedClientKey, DeleteClientKey, SetClientKeyEnabled, SortDirection, UpdateClientKey,
|
||||
},
|
||||
use gateway_admin::model::client_keys::{
|
||||
ClientKeyCursor, ClientKeyCursorValue as DomainCursorValue, ClientKeyListQuery,
|
||||
ClientKeyMutation, ClientKeyPage, ClientKeyPageSize, ClientKeyRecord, ClientKeySecret,
|
||||
ClientKeySort as DomainSort, ClientKeySortField as DomainSortField, CreateClientKey,
|
||||
CreatedClientKey, DeleteClientKey, SetClientKeyEnabled, SortDirection, UpdateClientKey,
|
||||
};
|
||||
use gateway_core::{
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
@@ -138,7 +135,6 @@ impl ClientKeySort {
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CreateClientKeyRequest {
|
||||
expected_config_revision: u64,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
@@ -149,14 +145,12 @@ pub struct CreateClientKeyRequest {
|
||||
impl CreateClientKeyRequest {
|
||||
/// 校验 wire 边界并直接构造管理用例命令。
|
||||
pub fn into_command(self) -> Result<CreateClientKey, WireValidationError> {
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(CreateClientKey {
|
||||
expected_config_revision: revision(self.expected_config_revision)?,
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
provider_kind: ProviderKind::new(self.provider_kind)
|
||||
@@ -174,7 +168,6 @@ impl CreateClientKeyRequest {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateClientKeyRequest {
|
||||
id: String,
|
||||
expected_config_revision: u64,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
@@ -186,14 +179,12 @@ impl UpdateClientKeyRequest {
|
||||
/// 校验 wire 边界并直接构造管理用例命令。
|
||||
pub fn into_command(self) -> Result<UpdateClientKey, WireValidationError> {
|
||||
validate_required_text(&self.id, "id")?;
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(UpdateClientKey {
|
||||
expected_config_revision: revision(self.expected_config_revision)?,
|
||||
id: client_key_id(self.id, "clientKeyMutationNotFound")?,
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
@@ -207,12 +198,11 @@ impl UpdateClientKeyRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只携带 ID 与配置 revision 的 Client Key mutation 请求。
|
||||
/// 只携带 ID 的 Client Key mutation 请求。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct ClientKeyRevisionRequest {
|
||||
pub struct ClientKeyMutationRequest {
|
||||
id: String,
|
||||
expected_config_revision: u64,
|
||||
}
|
||||
|
||||
/// 读取一次完整 Key 的 ID query。
|
||||
@@ -233,20 +223,15 @@ impl ClientKeyIdQuery {
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientKeyRevisionRequest {
|
||||
/// 校验请求并取出 ID 与 revision。
|
||||
pub fn into_parts(self) -> Result<(String, u64), WireValidationError> {
|
||||
impl ClientKeyMutationRequest {
|
||||
/// 校验请求并取出 ID。
|
||||
pub fn into_id(self) -> Result<String, WireValidationError> {
|
||||
validate_required_text(&self.id, "id")?;
|
||||
validate_revision(self.expected_config_revision)?;
|
||||
Ok((self.id, self.expected_config_revision))
|
||||
Ok(self.id)
|
||||
}
|
||||
|
||||
fn into_command_parts(self) -> Result<(ClientApiKeyId, Revision), WireValidationError> {
|
||||
let (id, expected_config_revision) = self.into_parts()?;
|
||||
Ok((
|
||||
client_key_id(id, "clientKeyMutationNotFound")?,
|
||||
revision(expected_config_revision)?,
|
||||
))
|
||||
fn into_domain_id(self) -> Result<ClientApiKeyId, WireValidationError> {
|
||||
client_key_id(self.into_id()?, "clientKeyMutationNotFound")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,7 +274,6 @@ impl From<ClientKeyRecord> for ClientKeyView {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ClientKeyListData {
|
||||
config_revision: u64,
|
||||
items: Vec<ClientKeyView>,
|
||||
next_cursor: Option<String>,
|
||||
total: u64,
|
||||
@@ -298,14 +282,8 @@ pub struct ClientKeyListData {
|
||||
impl ClientKeyListData {
|
||||
/// 构造 Client Key 列表响应。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
config_revision: u64,
|
||||
items: Vec<ClientKeyView>,
|
||||
next_cursor: Option<String>,
|
||||
total: u64,
|
||||
) -> Self {
|
||||
pub fn new(items: Vec<ClientKeyView>, next_cursor: Option<String>, total: u64) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
items,
|
||||
next_cursor,
|
||||
total,
|
||||
@@ -325,7 +303,6 @@ impl TryFrom<ClientKeyPage> for ClientKeyListData {
|
||||
.map(encode_client_key_cursor)
|
||||
.transpose()?;
|
||||
Ok(Self::new(
|
||||
page.config_revision.get(),
|
||||
page.items.into_iter().map(Into::into).collect(),
|
||||
next_cursor,
|
||||
page.total,
|
||||
@@ -337,7 +314,6 @@ impl TryFrom<ClientKeyPage> for ClientKeyListData {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CreatedClientKeyData {
|
||||
config_revision: u64,
|
||||
id: String,
|
||||
prefix: String,
|
||||
plaintext_key: String,
|
||||
@@ -346,9 +322,8 @@ pub struct CreatedClientKeyData {
|
||||
impl CreatedClientKeyData {
|
||||
/// 构造一次性创建响应。
|
||||
#[must_use]
|
||||
pub fn new(config_revision: u64, id: String, prefix: String, plaintext_key: String) -> Self {
|
||||
pub fn new(id: String, prefix: String, plaintext_key: String) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
id,
|
||||
prefix,
|
||||
plaintext_key,
|
||||
@@ -359,7 +334,6 @@ impl CreatedClientKeyData {
|
||||
impl From<CreatedClientKey> for CreatedClientKeyData {
|
||||
fn from(created: CreatedClientKey) -> Self {
|
||||
Self::new(
|
||||
created.config_revision.get(),
|
||||
created.secret.record.id.to_string(),
|
||||
created.secret.record.prefix.clone(),
|
||||
created.secret.expose_for_response().to_owned(),
|
||||
@@ -371,7 +345,6 @@ impl fmt::Debug for CreatedClientKeyData {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CreatedClientKeyData")
|
||||
.field("config_revision", &self.config_revision)
|
||||
.field("id", &self.id)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("plaintext_key", &"[REDACTED]")
|
||||
@@ -417,24 +390,20 @@ impl fmt::Debug for RevealedClientKeyData {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MutatedClientKeyData {
|
||||
config_revision: u64,
|
||||
id: String,
|
||||
}
|
||||
|
||||
impl MutatedClientKeyData {
|
||||
/// 构造 mutation 响应。
|
||||
#[must_use]
|
||||
pub fn new(config_revision: u64, id: String) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
id,
|
||||
}
|
||||
pub fn new(id: String) -> Self {
|
||||
Self { id }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ClientKeyMutation> for MutatedClientKeyData {
|
||||
fn from(mutation: ClientKeyMutation) -> Self {
|
||||
Self::new(mutation.config_revision.get(), mutation.id.to_string())
|
||||
Self::new(mutation.id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -582,18 +551,6 @@ fn wire_cursor(cursor: ClientKeyCursor) -> Result<ClientKeyCursorData, WireValid
|
||||
Ok(cursor)
|
||||
}
|
||||
|
||||
fn validate_revision(value: u64) -> Result<(), WireValidationError> {
|
||||
if value == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn revision(value: u64) -> Result<Revision, WireValidationError> {
|
||||
validate_revision(value)?;
|
||||
Revision::new(value).map_err(|_| WireValidationError::new("expectedConfigRevision"))
|
||||
}
|
||||
|
||||
fn client_key_id(
|
||||
value: String,
|
||||
field: &'static str,
|
||||
@@ -768,23 +725,19 @@ where
|
||||
async fn disable_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.set_enabled(
|
||||
&auth.context().mutation_context(),
|
||||
SetClientKeyEnabled {
|
||||
expected_config_revision,
|
||||
id,
|
||||
enabled: false,
|
||||
},
|
||||
SetClientKeyEnabled { id, enabled: false },
|
||||
)
|
||||
.await,
|
||||
)
|
||||
@@ -793,23 +746,19 @@ where
|
||||
async fn enable_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.set_enabled(
|
||||
&auth.context().mutation_context(),
|
||||
SetClientKeyEnabled {
|
||||
expected_config_revision,
|
||||
id,
|
||||
enabled: true,
|
||||
},
|
||||
SetClientKeyEnabled { id, enabled: true },
|
||||
)
|
||||
.await,
|
||||
)
|
||||
@@ -818,23 +767,17 @@ where
|
||||
async fn delete_client_key<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(payload): Json<ClientKeyRevisionRequest>,
|
||||
Json(payload): Json<ClientKeyMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (id, expected_config_revision) = payload.into_command_parts().map_err(map_wire_error)?;
|
||||
let id = payload.into_domain_id().map_err(map_wire_error)?;
|
||||
mutation_response(
|
||||
state
|
||||
.admin_services()
|
||||
.client_keys()
|
||||
.delete(
|
||||
&auth.context().mutation_context(),
|
||||
DeleteClientKey {
|
||||
expected_config_revision,
|
||||
id,
|
||||
},
|
||||
)
|
||||
.delete(&auth.context().mutation_context(), DeleteClientKey { id })
|
||||
.await,
|
||||
)
|
||||
}
|
||||
@@ -854,9 +797,6 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
"cursor" => AdminError::bad_request("Invalid client key cursor"),
|
||||
"clientKeyRevealNotFound" => AdminError::not_found("Client API key was not found"),
|
||||
"clientKeyMutationNotFound" => AdminError::not_found("client API key was not found"),
|
||||
"expectedConfigRevision" => {
|
||||
AdminError::bad_request("expectedConfigRevision must be positive")
|
||||
}
|
||||
_ => AdminError::bad_request("Invalid client API key request"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -801,8 +801,8 @@ pub struct DashboardCardsView {
|
||||
pub struct DashboardCredentialsCardView {
|
||||
pub total: String,
|
||||
pub total_value: u64,
|
||||
pub enabled: String,
|
||||
pub enabled_value: u64,
|
||||
pub available: String,
|
||||
pub available_value: u64,
|
||||
pub unavailable: String,
|
||||
pub unavailable_value: u64,
|
||||
}
|
||||
@@ -1235,7 +1235,6 @@ pub struct OpsErrorView {
|
||||
pub account_id: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
@@ -1954,7 +1953,6 @@ fn usage_page_view(
|
||||
}
|
||||
|
||||
fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
let status = error.status_code.map(i64::from);
|
||||
OpsErrorView {
|
||||
id: error.event_id,
|
||||
request_id: error.request_id,
|
||||
@@ -1964,9 +1962,8 @@ fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
account_id: error.provider_account_ref,
|
||||
route: error.operation,
|
||||
model: error.upstream_model_id,
|
||||
status_code: status,
|
||||
client_status_code: None,
|
||||
upstream_status_code: None,
|
||||
client_status_code: error.client_status_code.map(i64::from),
|
||||
upstream_status_code: error.upstream_status_code.map(i64::from),
|
||||
transport: error.upstream_transport,
|
||||
attempt_index: error.attempt_index,
|
||||
failure_class: error.failure_kind,
|
||||
@@ -2290,8 +2287,8 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
credentials: DashboardCredentialsCardView {
|
||||
total: format_compact_number(provider_accounts.total),
|
||||
total_value: provider_accounts.total,
|
||||
enabled: format_compact_number(provider_accounts.enabled),
|
||||
enabled_value: provider_accounts.enabled,
|
||||
available: format_compact_number(provider_accounts.active),
|
||||
available_value: provider_accounts.active,
|
||||
unavailable: format_compact_number(provider_accounts.unavailable),
|
||||
unavailable_value: provider_accounts.unavailable,
|
||||
},
|
||||
|
||||
@@ -10,12 +10,8 @@ use axum::{
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
settings::{
|
||||
ModelMappings as DomainModelMappings, ReplaceRuntimeSettings, RotationStrategy,
|
||||
RuntimeSettings,
|
||||
},
|
||||
use gateway_admin::model::settings::{
|
||||
ModelMappings as DomainModelMappings, ReplaceRuntimeSettings, RotationStrategy, RuntimeSettings,
|
||||
};
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -32,7 +28,6 @@ pub type ModelMappings = BTreeMap<String, String>;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
pub config_revision: u64,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -49,7 +44,6 @@ pub struct RuntimeSettingsView {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateRuntimeSettingsRequest {
|
||||
pub expected_config_revision: u64,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
@@ -64,9 +58,6 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
/// 校验公共运行参数。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
if self.expected_config_revision == 0 {
|
||||
return Err(WireValidationError::new("expectedConfigRevision"));
|
||||
}
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
for (value, field) in [
|
||||
(self.refresh_margin_seconds, "refreshMarginSeconds"),
|
||||
@@ -93,8 +84,6 @@ impl UpdateRuntimeSettingsRequest {
|
||||
fn into_command(self) -> Result<ReplaceRuntimeSettings, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(ReplaceRuntimeSettings {
|
||||
expected_config_revision: Revision::new(self.expected_config_revision)
|
||||
.map_err(|_| WireValidationError::new("settingsRevisionOverflow"))?,
|
||||
model_mappings: domain_model_mappings(self.model_mappings)?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
@@ -117,7 +106,6 @@ impl UpdateRuntimeSettingsRequest {
|
||||
impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
fn from(settings: RuntimeSettings) -> Self {
|
||||
Self {
|
||||
config_revision: settings.config_revision.get(),
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
@@ -358,7 +346,6 @@ fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRevisionOverflow" => "expectedConfigRevision must be positive".to_owned(),
|
||||
"settingsRefreshConcurrencyOverflow" => "Invalid refreshConcurrency".to_owned(),
|
||||
"settingsMaxConcurrencyOverflow" => "Invalid maxConcurrentPerAccount".to_owned(),
|
||||
"settingsUsageRetentionOverflow" => "Invalid usageRetentionDays".to_owned(),
|
||||
|
||||
@@ -201,11 +201,13 @@ struct RestartAcceptedView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SystemUpdateEventView {
|
||||
id: String,
|
||||
operation_id: Option<String>,
|
||||
level: &'static str,
|
||||
step: Option<String>,
|
||||
message: String,
|
||||
terminal: bool,
|
||||
progress_percent: Option<u8>,
|
||||
at: String,
|
||||
}
|
||||
|
||||
@@ -403,11 +405,13 @@ where
|
||||
impl From<SystemUpdateEvent> for SystemUpdateEventView {
|
||||
fn from(event: SystemUpdateEvent) -> Self {
|
||||
Self {
|
||||
id: event.id,
|
||||
operation_id: event.operation_id,
|
||||
level: update_event_level_name(event.level),
|
||||
step: event.step,
|
||||
message: event.message,
|
||||
terminal: event.terminal,
|
||||
progress_percent: event.progress_percent,
|
||||
at: event.occurred_at.to_rfc3339(),
|
||||
}
|
||||
}
|
||||
@@ -416,11 +420,13 @@ impl From<SystemUpdateEvent> for SystemUpdateEventView {
|
||||
impl SystemUpdateEventView {
|
||||
fn into_json(self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"id": self.id,
|
||||
"operationId": self.operation_id,
|
||||
"level": self.level,
|
||||
"step": self.step,
|
||||
"message": self.message,
|
||||
"terminal": self.terminal,
|
||||
"progressPercent": self.progress_percent,
|
||||
"at": self.at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use gateway_core::engine::execution::ExecutionService;
|
||||
use gateway_core::health::{HealthProbe, WorkerHealthSource};
|
||||
use gateway_core::lifecycle::ConnectionLifecycle;
|
||||
use serde::Deserialize;
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
use tower_http::timeout::TimeoutLayer;
|
||||
@@ -146,11 +146,17 @@ pub fn initialize(
|
||||
.map_err(|_| ApiError::Config(ApiConfigError::InvalidCorsOrigin))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
// credentials 模式禁止通配 allow_headers;显式列出鉴权与内容协商头。
|
||||
router = router.layer(
|
||||
CorsLayer::new()
|
||||
.allow_origin(origins)
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_headers(Any)
|
||||
.allow_headers([
|
||||
axum::http::header::AUTHORIZATION,
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
HeaderName::from_static("x-api-key"),
|
||||
request_id_header.clone(),
|
||||
])
|
||||
.allow_credentials(true),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -75,10 +75,14 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
if error.kind() == ProviderErrorKind::Unavailable
|
||||
&& error.send_state() == UpstreamSendState::NotSent =>
|
||||
{
|
||||
GatewayError::new(
|
||||
let gateway = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
)
|
||||
);
|
||||
match error.client_visible_upstream_error() {
|
||||
Some(upstream) => gateway.with_client_visible_upstream_error(upstream.clone()),
|
||||
None => gateway,
|
||||
}
|
||||
}
|
||||
EngineError::Provider(error) => GatewayError::from_provider(error),
|
||||
EngineError::Cancelled => {
|
||||
@@ -98,6 +102,7 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
| EngineError::ContinuationPinMismatch
|
||||
| EngineError::RequiredAccountMismatch
|
||||
| EngineError::DownstreamCommitRequired
|
||||
| EngineError::NoActiveAttempt
|
||||
| EngineError::InvalidDeliveryState => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
}
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
extract::{Path, Query, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{PublicModelId, PublicModelProfile};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::ApiState;
|
||||
@@ -17,20 +18,50 @@ use super::{
|
||||
};
|
||||
|
||||
const MODEL_CREATED_TIMESTAMP: i64 = 1_700_000_000;
|
||||
const CODEX_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent. Follow the user's instructions and use the available tools to complete software engineering tasks. Inspect relevant files before editing, preserve unrelated changes, and verify the result.";
|
||||
|
||||
/// `GET /v1/models`。
|
||||
pub(crate) async fn models(State(state): State<ApiState>, headers: HeaderMap) -> Response {
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ModelsQuery {
|
||||
client_version: Option<String>,
|
||||
}
|
||||
|
||||
/// `GET /v1/models`。Codex 携带 `client_version` 时返回其专用目录合同。
|
||||
pub(crate) async fn models(
|
||||
State(state): State<ApiState>,
|
||||
Query(query): Query<ModelsQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return authentication_error_response(error),
|
||||
};
|
||||
|
||||
if query
|
||||
.client_version
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.trim().is_empty())
|
||||
{
|
||||
let models = service
|
||||
.public_model_profiles(&client)
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, profile)| codex_model_json(profile, index))
|
||||
.collect::<Vec<_>>();
|
||||
return (
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"models": models,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let data = service
|
||||
.public_models(&client)
|
||||
.into_iter()
|
||||
.map(|model| openai_model_json(&model))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
@@ -134,3 +165,92 @@ fn catalog_model_json(id: &str) -> Value {
|
||||
"source": "gateway",
|
||||
})
|
||||
}
|
||||
|
||||
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
let id = profile.model().as_str();
|
||||
let presentation = profile.presentation();
|
||||
let reasoning_levels = presentation
|
||||
.supported_reasoning_efforts()
|
||||
.iter()
|
||||
.map(|effort| {
|
||||
json!({
|
||||
"effort": effort,
|
||||
"description": reasoning_effort_description(effort),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let reasoning_supported = presentation
|
||||
.supported_reasoning_efforts()
|
||||
.iter()
|
||||
.any(|effort| effort != "none");
|
||||
let context_window = presentation.context_window_tokens();
|
||||
let input_modalities = if presentation.image_input() {
|
||||
vec!["text", "image"]
|
||||
} else {
|
||||
vec!["text"]
|
||||
};
|
||||
let additional_speed_tiers = presentation
|
||||
.service_tiers()
|
||||
.iter()
|
||||
.filter_map(|tier| tier.speed_tier())
|
||||
.collect::<Vec<_>>();
|
||||
let service_tiers = presentation
|
||||
.service_tiers()
|
||||
.iter()
|
||||
.map(|tier| {
|
||||
json!({
|
||||
"id": tier.id(),
|
||||
"name": tier.name(),
|
||||
"description": tier.description(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
json!({
|
||||
"slug": id,
|
||||
"display_name": presentation.display_name().unwrap_or(id),
|
||||
"description": presentation.description(),
|
||||
"default_reasoning_level": presentation.default_reasoning_effort(),
|
||||
"supported_reasoning_levels": reasoning_levels,
|
||||
"shell_type": "shell_command",
|
||||
"visibility": if presentation.hidden() { "hide" } else { "list" },
|
||||
"supported_in_api": true,
|
||||
"priority": index.saturating_add(1),
|
||||
"additional_speed_tiers": additional_speed_tiers,
|
||||
"service_tiers": service_tiers,
|
||||
"availability_nux": Value::Null,
|
||||
"upgrade": Value::Null,
|
||||
"base_instructions": CODEX_BASE_INSTRUCTIONS,
|
||||
"model_messages": Value::Null,
|
||||
"include_skills_usage_instructions": false,
|
||||
"supports_reasoning_summary_parameter": reasoning_supported,
|
||||
"default_reasoning_summary": "auto",
|
||||
"support_verbosity": presentation.verbosity(),
|
||||
"default_verbosity": Value::Null,
|
||||
"apply_patch_tool_type": presentation.agent_tools().then_some("freeform"),
|
||||
"web_search_tool_type": "text",
|
||||
"truncation_policy": { "mode": "tokens", "limit": 10_000 },
|
||||
"supports_parallel_tool_calls": presentation.parallel_tool_calls(),
|
||||
"supports_image_detail_original": presentation.image_detail_original(),
|
||||
"context_window": context_window,
|
||||
"max_context_window": context_window,
|
||||
"effective_context_window_percent": 95,
|
||||
"experimental_supported_tools": [],
|
||||
"input_modalities": input_modalities,
|
||||
"supports_search_tool": presentation.search_tool(),
|
||||
"use_responses_lite": false,
|
||||
})
|
||||
}
|
||||
|
||||
fn reasoning_effort_description(effort: &str) -> &'static str {
|
||||
match effort {
|
||||
"none" => "No reasoning",
|
||||
"minimal" => "Minimal reasoning",
|
||||
"low" => "Fast responses with lighter reasoning",
|
||||
"medium" => "Balances speed and reasoning depth for everyday tasks",
|
||||
"high" => "Greater reasoning depth for complex problems",
|
||||
"xhigh" => "Extra high reasoning depth for complex problems",
|
||||
"max" => "Maximum reasoning depth for the hardest problems",
|
||||
_ => "Provider-supported reasoning level",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,9 +310,18 @@ pub(super) fn decode_request_inner(
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
let value =
|
||||
serde_json::from_slice::<Value>(body).map_err(|_| RequestDecodeError::MalformedJson)?;
|
||||
let Value::Object(mut object) = value else {
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
decode_request_object(object, review, request_headers)
|
||||
}
|
||||
|
||||
/// 解码已解析的顶层 object;WebSocket 帧解码复用此内核,避免序列化往返。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// `use_websocket` 只影响本地 transport。无论值是否可识别,都不得进入上游 body。
|
||||
let use_websocket = object
|
||||
.remove("use_websocket")
|
||||
|
||||
@@ -549,9 +549,7 @@ fn decode_response_create_inner(
|
||||
if matches!(body.get("stream"), Some(value) if value.as_bool() != Some(true)) {
|
||||
return Err(ResponseCreateFrameError::StreamingRequired);
|
||||
}
|
||||
let encoded = serde_json::to_vec(&Value::Object(body))
|
||||
.map_err(|_| ResponseCreateFrameError::InvalidJson)?;
|
||||
super::request::decode_request_inner(&encoded, false, request_headers)
|
||||
super::request::decode_request_object(body, false, request_headers)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ use gateway_core::engine::execution::{
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLifecycle};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{PublicModelId, PublicModelProfile};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::auth::ClientApiKeyAuthError;
|
||||
@@ -51,6 +51,13 @@ impl OpenAiService {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn public_model_profiles(
|
||||
&self,
|
||||
client: &AuthenticatedClient,
|
||||
) -> Vec<PublicModelProfile> {
|
||||
self.execution.public_model_profiles(client)
|
||||
}
|
||||
|
||||
pub(crate) fn contains_public_model(
|
||||
&self,
|
||||
client: &AuthenticatedClient,
|
||||
|
||||
@@ -147,7 +147,7 @@ mod actions {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_actions_should_require_frozen_account_ids_and_revision() {
|
||||
fn account_actions_should_require_frozen_account_ids_and_reject_unknown_revision() {
|
||||
let id: AccountIdQuery =
|
||||
serde_json::from_value(json!({ "accountId": "acct_1" })).expect("decode ID query");
|
||||
assert!(id.validate().is_ok());
|
||||
@@ -155,13 +155,16 @@ mod actions {
|
||||
serde_json::from_value(json!({ "accountId": "legacy-id" })).expect("decode action");
|
||||
assert_eq!(action.validate().unwrap_err().field(), "accountId");
|
||||
let refresh: AccountRefreshRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
"accountId": "acct_1"
|
||||
}))
|
||||
.expect("decode refresh");
|
||||
assert_eq!(
|
||||
refresh.validate().unwrap_err().field(),
|
||||
"expectedConfigRevision"
|
||||
assert!(refresh.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<AccountRefreshRequest>(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -169,7 +172,6 @@ mod actions {
|
||||
fn account_import_should_use_provider_and_opaque_data_fields() {
|
||||
let valid: AccountImportRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": {
|
||||
"providerOwnedUnknownField": {"nested": [1, 2, 3]},
|
||||
"accounts": [{"credentials": {"access_token": "provider-validates-this"}}]
|
||||
@@ -180,7 +182,6 @@ mod actions {
|
||||
|
||||
let invalid: AccountImportRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": []
|
||||
}))
|
||||
.expect("decode invalid account import");
|
||||
@@ -188,27 +189,32 @@ mod actions {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"document": {}
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"data": {}
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_deletion_should_validate_one_provider_batch_and_emit_account_ids() {
|
||||
let request: AccountDeletionRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"accountIds": ["acct_1", "acct_2"],
|
||||
"expectedConfigRevision": 8
|
||||
"accountIds": ["acct_1", "acct_2"]
|
||||
}))
|
||||
.expect("decode account deletion");
|
||||
assert!(request.validate().is_ok());
|
||||
|
||||
let duplicate: AccountDeletionRequest = serde_json::from_value(json!({
|
||||
"provider": "xai",
|
||||
"accountIds": ["acct_1", "acct_1"],
|
||||
"expectedConfigRevision": 8
|
||||
"accountIds": ["acct_1", "acct_1"]
|
||||
}))
|
||||
.expect("decode duplicate account deletion");
|
||||
assert_eq!(duplicate.validate().unwrap_err().field(), "accountIds");
|
||||
@@ -223,7 +229,6 @@ mod actions {
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize account deletion"),
|
||||
json!({
|
||||
"configRevision": 9,
|
||||
"deletedCount": 2,
|
||||
"accountIds": ["acct_1", "acct_2"]
|
||||
})
|
||||
@@ -239,7 +244,6 @@ mod actions {
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize account import"),
|
||||
json!({
|
||||
"configRevision": 8,
|
||||
"importedCount": 1,
|
||||
"accountIds": ["acct_imported"]
|
||||
})
|
||||
@@ -276,6 +280,8 @@ mod actions {
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
provider_error_type: Some("invalid_request_error".to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
},
|
||||
]
|
||||
@@ -302,7 +308,13 @@ mod actions {
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "error", "error": "upstream unavailable", "accountStatus": "active" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
"providerErrorCode": "usage_exhausted",
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"accountStatus": "active"
|
||||
}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -51,7 +51,6 @@ async fn default_auth_service_should_initialize_login_validate_and_logout() {
|
||||
let service = fixture.services.auth();
|
||||
let session = service
|
||||
.login(LoginCommand {
|
||||
source: "127.0.0.1".to_owned(),
|
||||
username: Some("admin_1".to_owned()),
|
||||
password: "strong-admin-password".to_owned(),
|
||||
})
|
||||
@@ -84,42 +83,6 @@ async fn default_auth_service_should_initialize_login_validate_and_logout() {
|
||||
assert_eq!(fixture.auth.audit_count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_auth_service_should_throttle_repeated_invalid_identity() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
for attempt in 1..=5 {
|
||||
let error = fixture
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "shared-source".to_owned(),
|
||||
username: Some("wrong-user".to_owned()),
|
||||
password: "wrong-password".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect_err("invalid login");
|
||||
if attempt < 5 {
|
||||
assert_eq!(error, LoginError::InvalidCredentials);
|
||||
} else {
|
||||
assert_eq!(error, LoginError::Throttled);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
fixture
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "shared-source".to_owned(),
|
||||
username: None,
|
||||
password: "anything".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect_err("source remains throttled"),
|
||||
LoginError::Throttled
|
||||
);
|
||||
assert_eq!(fixture.auth.session_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_auth_service_should_verify_only_full_plaintext_admin_key() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
@@ -162,7 +125,6 @@ async fn audit_failure_should_revoke_new_session_before_returning_it() {
|
||||
.services
|
||||
.auth()
|
||||
.login(LoginCommand {
|
||||
source: "source".to_owned(),
|
||||
username: None,
|
||||
password: "strong-admin-password".to_owned(),
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use chrono::{DateTime, TimeZone as _, Utc};
|
||||
use gateway_api::admin::client_keys::{
|
||||
self, ClientKeyCursorData, ClientKeyCursorValue, ClientKeyListData, ClientKeyRevisionRequest,
|
||||
self, ClientKeyCursorData, ClientKeyCursorValue, ClientKeyListData, ClientKeyMutationRequest,
|
||||
ClientKeySort, ClientKeySortDirection, ClientKeySortField, ClientKeyView,
|
||||
CreateClientKeyRequest, CreatedClientKeyData, ListClientKeysQuery, MutatedClientKeyData,
|
||||
RevealedClientKeyData, UpdateClientKeyRequest, decode_client_key_cursor,
|
||||
@@ -101,9 +101,8 @@ fn client_key_queries_should_reject_unknown_zero_and_oversized_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields() {
|
||||
fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
let valid = serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"name": "terminal key",
|
||||
"label": "production",
|
||||
"providerKind": "openai",
|
||||
@@ -113,22 +112,11 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
.expect("deserialize create")
|
||||
.into_command()
|
||||
.expect("validate create");
|
||||
assert_eq!(valid.expected_config_revision.get(), 7);
|
||||
assert_eq!(valid.name, "terminal key");
|
||||
|
||||
for (payload, field) in [
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 0,
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0
|
||||
}),
|
||||
"expectedConfigRevision",
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 1,
|
||||
"name": " ",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 0,
|
||||
@@ -138,7 +126,6 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"expectedConfigRevision": 1,
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": u64::MAX,
|
||||
@@ -170,14 +157,23 @@ fn client_key_mutations_should_validate_revision_text_limits_and_unknown_fields(
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
let revision = serde_json::from_value::<ClientKeyRevisionRequest>(json!({
|
||||
"id": "key_1",
|
||||
"expectedConfigRevision": 3
|
||||
assert!(
|
||||
serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"name": "terminal key",
|
||||
"providerKind": "openai",
|
||||
"maxConcurrency": 2,
|
||||
"requestsPerMinute": 60
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
let mutation_id = serde_json::from_value::<ClientKeyMutationRequest>(json!({
|
||||
"id": "key_1"
|
||||
}))
|
||||
.expect("deserialize revision mutation")
|
||||
.into_parts()
|
||||
.expect("validate revision mutation");
|
||||
assert_eq!(revision, ("key_1".to_owned(), 3));
|
||||
.expect("deserialize mutation")
|
||||
.into_id()
|
||||
.expect("validate mutation");
|
||||
assert_eq!(mutation_id, "key_1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -225,9 +221,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
updated_at: created_at,
|
||||
last_used_at: Some(created_at),
|
||||
});
|
||||
let list = serde_json::to_value(ClientKeyListData::new(9, vec![view], None, 1))
|
||||
.expect("serialize list");
|
||||
assert_eq!(list["configRevision"], 9);
|
||||
let list =
|
||||
serde_json::to_value(ClientKeyListData::new(vec![view], None, 1)).expect("serialize list");
|
||||
assert!(list.get("configRevision").is_none());
|
||||
assert_eq!(list["total"], 1);
|
||||
assert_eq!(list["items"][0]["id"], "key_visible");
|
||||
assert_eq!(list["items"][0]["providerKind"], "openai");
|
||||
@@ -248,7 +244,6 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
|
||||
let plaintext = format!("sk_{}", "a".repeat(43));
|
||||
let created = CreatedClientKeyData::new(
|
||||
10,
|
||||
"key_created".to_owned(),
|
||||
"sk_aaaaaaaaa".to_owned(),
|
||||
plaintext.clone(),
|
||||
@@ -263,9 +258,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
json!({ "id": "key_created", "plaintextKey": plaintext })
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(MutatedClientKeyData::new(11, "key_created".to_owned()))
|
||||
serde_json::to_value(MutatedClientKeyData::new("key_created".to_owned()))
|
||||
.expect("serialize mutation"),
|
||||
json!({ "configRevision": 11, "id": "key_created" })
|
||||
json!({ "id": "key_created" })
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -84,6 +84,10 @@ pub(super) struct AdminTestFixture {
|
||||
|
||||
impl AdminTestFixture {
|
||||
pub async fn new() -> Self {
|
||||
Self::with_system(Arc::new(UnusedSystem)).await
|
||||
}
|
||||
|
||||
pub async fn with_system(system: Arc<dyn SystemOperations>) -> Self {
|
||||
let api_key = Arc::new(Mutex::new(None));
|
||||
let auth = Arc::new(MemoryAuthStore::new(api_key.clone()));
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
@@ -113,7 +117,7 @@ impl AdminTestFixture {
|
||||
providers,
|
||||
Arc::new(NoopSnapshot),
|
||||
Arc::new(NoopProbe),
|
||||
Arc::new(UnusedSystem),
|
||||
system,
|
||||
)
|
||||
.await
|
||||
.expect("initialize test admin services");
|
||||
@@ -142,7 +146,6 @@ impl AdminSessionState for AdminTestState {
|
||||
pub(super) struct MemoryAuthStore {
|
||||
password_hash: Mutex<Option<String>>,
|
||||
sessions: Mutex<BTreeMap<String, AdminSession>>,
|
||||
failures: Mutex<BTreeMap<String, u32>>,
|
||||
audits: Mutex<Vec<AdminAuditEvent>>,
|
||||
api_key: Arc<Mutex<Option<AdminApiKey>>>,
|
||||
fail_audit: AtomicBool,
|
||||
@@ -153,7 +156,6 @@ impl MemoryAuthStore {
|
||||
Self {
|
||||
password_hash: Mutex::new(None),
|
||||
sessions: Mutex::new(BTreeMap::new()),
|
||||
failures: Mutex::new(BTreeMap::new()),
|
||||
audits: Mutex::new(Vec::new()),
|
||||
api_key,
|
||||
fail_audit: AtomicBool::new(false),
|
||||
@@ -235,37 +237,6 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(self.sessions.lock().expect("sessions").remove(session_id))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(self
|
||||
.failures
|
||||
.lock()
|
||||
.expect("failures")
|
||||
.get(source)
|
||||
.is_some_and(|count| *count >= failure_limit))
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
_: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut failures = self.failures.lock().expect("failures");
|
||||
let count = failures.entry(source.to_owned()).or_default();
|
||||
*count = count.saturating_add(1);
|
||||
Ok(*count >= failure_limit)
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> AdminStoreResult<()> {
|
||||
self.failures.lock().expect("failures").remove(source);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
if self.fail_audit.load(Ordering::SeqCst) {
|
||||
return Err(unavailable("auth audit"));
|
||||
@@ -308,8 +279,9 @@ impl SettingsStore for MemorySettingsStore {
|
||||
command: ReplaceRuntimeSettings,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
let updated = RuntimeSettings {
|
||||
config_revision: next_revision(command.expected_config_revision),
|
||||
config_revision: next_revision(settings.config_revision),
|
||||
model_mappings: command.model_mappings,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
@@ -321,31 +293,31 @@ impl SettingsStore for MemorySettingsStore {
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
*self.settings.lock().expect("settings") = updated.clone();
|
||||
*settings = updated.clone();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
key: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
*self.api_key.lock().expect("API key") = Some(key);
|
||||
let revision = self.settings.lock().expect("settings").config_revision;
|
||||
Ok(AdminApiKeyMutation {
|
||||
config_revision: next_revision(expected_config_revision),
|
||||
config_revision: next_revision(revision),
|
||||
exists: true,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
*self.api_key.lock().expect("API key") = None;
|
||||
let revision = self.settings.lock().expect("settings").config_revision;
|
||||
Ok(AdminApiKeyMutation {
|
||||
config_revision: next_revision(expected_config_revision),
|
||||
config_revision: next_revision(revision),
|
||||
exists: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -37,7 +37,6 @@ async fn response_json(response: axum::response::Response) -> Value {
|
||||
|
||||
fn update_body() -> Value {
|
||||
json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
@@ -64,22 +63,11 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_require_positive_revision() {
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
let mut body = update_body();
|
||||
body["expectedConfigRevision"] = json!(0);
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(body).expect("decode settings");
|
||||
assert_eq!(
|
||||
request.validate().unwrap_err().field(),
|
||||
"expectedConfigRevision"
|
||||
);
|
||||
body["expectedConfigRevision"] = json!(7);
|
||||
|
||||
let mut missing = update_body();
|
||||
missing
|
||||
.as_object_mut()
|
||||
.expect("settings object")
|
||||
.remove("expectedConfigRevision");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(missing).is_err());
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -124,7 +112,7 @@ async fn settings_post_should_replace_global_model_mappings() {
|
||||
.expect("settings update response");
|
||||
let data = response_json(response).await["data"].clone();
|
||||
|
||||
assert_eq!(data["configRevision"], 8);
|
||||
assert!(data.get("configRevision").is_none());
|
||||
assert_eq!(data["modelMappings"]["gpt-5.4"], "gpt-5.5");
|
||||
assert_eq!(data["modelMappings"]["grok-latest"], "grok-4.5");
|
||||
}
|
||||
@@ -213,3 +201,26 @@ async fn settings_should_accept_admin_api_key_header() {
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_auth_should_accept_a_configured_request_id_header_name() {
|
||||
use axum::http::HeaderName;
|
||||
use tower_http::request_id::{MakeRequestUuid, SetRequestIdLayer};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
// 部署把 api.request_id_header 改名后,注入的 header 不再叫 x-request-id;
|
||||
// 管理请求仍须拿到请求上下文,而不是退化为 500。
|
||||
let custom = HeaderName::from_static("x-trace-id");
|
||||
let app = app(fixture.state()).layer(SetRequestIdLayer::new(custom, MakeRequestUuid));
|
||||
let unlabelled = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri("/api/admin/settings")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.body(Body::empty())
|
||||
.expect("build settings request");
|
||||
|
||||
let response = app.oneshot(unlabelled).await.expect("settings response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,24 @@
|
||||
use gateway_api::admin::system::{UpdateDetailQuery, UpdateRequest};
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::stream;
|
||||
use gateway_admin::{
|
||||
model::system::{
|
||||
SystemOperationAccepted, SystemUpdateDetail, SystemUpdateEvent, SystemUpdateEventLevel,
|
||||
SystemUpdateStatus, SystemVersion,
|
||||
},
|
||||
ports::system::{SystemOperationError, SystemOperations, SystemUpdateEventStream},
|
||||
};
|
||||
use gateway_api::admin::system::{self, UpdateDetailQuery, UpdateRequest};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState, unavailable_system};
|
||||
|
||||
#[test]
|
||||
fn update_detail_query_should_default_to_no_refresh_and_reject_unknown_fields() {
|
||||
@@ -16,3 +36,84 @@ fn update_request_should_preserve_target_version_for_domain_validation() {
|
||||
serde_json::from_str::<UpdateRequest>(r#"{"targetVersion":"v0.2.0","extra":1}"#).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_event_stream_should_preserve_event_id_and_download_progress_percent() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem)).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/system/update/events")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_system_update_events")
|
||||
.body(Body::empty())
|
||||
.expect("update event request"),
|
||||
)
|
||||
.await
|
||||
.expect("update event response");
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("update event body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("UTF-8 event stream");
|
||||
|
||||
assert!(
|
||||
body.contains(r#""id":"update-progress-1""#),
|
||||
"unexpected update event stream: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.contains(r#""progressPercent":40"#),
|
||||
"unexpected update event stream: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
fn app(state: AdminTestState) -> Router {
|
||||
system::router::<AdminTestState>().with_state(state)
|
||||
}
|
||||
|
||||
struct ProgressSystem;
|
||||
|
||||
#[async_trait]
|
||||
impl SystemOperations for ProgressSystem {
|
||||
async fn version(&self) -> Result<SystemVersion, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn update_detail(&self, _: bool) -> Result<SystemUpdateDetail, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
fn update_events(&self) -> SystemUpdateEventStream {
|
||||
Box::pin(stream::once(async {
|
||||
SystemUpdateEvent {
|
||||
id: "update-progress-1".to_owned(),
|
||||
operation_id: Some("update-1".to_owned()),
|
||||
level: SystemUpdateEventLevel::Info,
|
||||
step: Some("download".to_owned()),
|
||||
message: "已下载 4.0 MiB / 10.0 MiB (40%)".to_owned(),
|
||||
terminal: true,
|
||||
progress_percent: Some(40),
|
||||
occurred_at: Utc::now(),
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn perform_update(
|
||||
&self,
|
||||
_: Option<String>,
|
||||
) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn update_status(&self) -> Result<SystemUpdateStatus, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn rollback(&self) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
|
||||
async fn restart(&self) -> Result<SystemOperationAccepted, SystemOperationError> {
|
||||
Err(unavailable_system())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,36 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"no account is eligible for the requested model",
|
||||
Some("no_eligible_account".to_owned()),
|
||||
Some("account_unavailable_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider detail");
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
);
|
||||
let gateway = gateway_error_from_engine(&error);
|
||||
|
||||
assert_eq!(gateway.kind(), GatewayErrorKind::NoAvailableProvider);
|
||||
assert_eq!(
|
||||
gateway.client_message(),
|
||||
"no account is eligible for the requested model"
|
||||
);
|
||||
assert_eq!(gateway.client_error_code(), Some("no_eligible_account"));
|
||||
assert_eq!(
|
||||
gateway.client_error_type(),
|
||||
Some("account_unavailable_error")
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"no upstream provider is currently available for this request"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
|
||||
@@ -39,11 +39,18 @@ use gateway_core::routing::{
|
||||
};
|
||||
|
||||
pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Router {
|
||||
api_router_with_origins(execution, Vec::new()).await
|
||||
}
|
||||
|
||||
pub(super) async fn api_router_with_origins(
|
||||
execution: Arc<dyn ExecutionService>,
|
||||
cors_allowed_origins: Vec<String>,
|
||||
) -> axum::Router {
|
||||
let admin = crate::admin::AdminTestFixture::new().await;
|
||||
gateway_api::initialize(
|
||||
gateway_api::ApiConfig {
|
||||
asset_directory: std::env::temp_dir(),
|
||||
cors_allowed_origins: Vec::new(),
|
||||
cors_allowed_origins,
|
||||
request_timeout_seconds: None,
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
},
|
||||
|
||||
@@ -10,19 +10,30 @@ use gateway_core::engine::execution::{
|
||||
StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::{
|
||||
ModelPresentation, ModelServiceTier, PublicModelId, PublicModelProfile,
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
struct ModelsExecution {
|
||||
pub(super) struct ModelsExecution {
|
||||
client: AuthenticatedClient,
|
||||
profiles: bool,
|
||||
}
|
||||
|
||||
impl ModelsExecution {
|
||||
fn new() -> Arc<Self> {
|
||||
pub(super) fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_models_test"),
|
||||
profiles: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn with_profiles() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_models_test"),
|
||||
profiles: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -46,6 +57,36 @@ impl ExecutionService for ModelsExecution {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn public_model_profiles(&self, _: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
if !self.profiles {
|
||||
return Vec::new();
|
||||
}
|
||||
vec![PublicModelProfile::new(
|
||||
PublicModelId::new("grok-4.5").expect("model"),
|
||||
ModelPresentation::new(
|
||||
Some("Grok 4.5".to_owned()),
|
||||
Some("xAI Grok 4.5 frontier model.".to_owned()),
|
||||
)
|
||||
.with_reasoning(
|
||||
Some("medium".to_owned()),
|
||||
["low", "medium", "high", "xhigh"]
|
||||
.map(str::to_owned)
|
||||
.to_vec(),
|
||||
)
|
||||
.with_context_window_tokens(Some(500_000))
|
||||
.with_image_input(true)
|
||||
.with_agent_tools(true, true)
|
||||
.with_service_tiers(vec![
|
||||
ModelServiceTier::new(
|
||||
"priority",
|
||||
"Fast",
|
||||
"Route the request through Codex fast mode.",
|
||||
)
|
||||
.with_speed_tier("fast"),
|
||||
]),
|
||||
)]
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
matches!(model.as_str(), "model-a" | "model-b")
|
||||
}
|
||||
@@ -91,6 +132,100 @@ async fn models_should_encode_the_service_visible_catalog() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_encode_provider_profiles_for_current_codex_clients() {
|
||||
let response = api_router(ModelsExecution::with_profiles())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models?client_version=0.145.0"))
|
||||
.await
|
||||
.expect("list Codex models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read Codex models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("models JSON");
|
||||
let model = &value["models"][0];
|
||||
|
||||
assert_eq!(
|
||||
value
|
||||
.as_object()
|
||||
.map(|catalog| catalog.keys().map(String::as_str).collect::<Vec<_>>()),
|
||||
Some(vec!["models"])
|
||||
);
|
||||
assert_eq!(model["slug"], "grok-4.5");
|
||||
assert_eq!(model["default_reasoning_level"], "medium");
|
||||
assert_eq!(model["context_window"], 500_000);
|
||||
assert_eq!(model["apply_patch_tool_type"], "freeform");
|
||||
assert_eq!(model["additional_speed_tiers"], serde_json::json!(["fast"]));
|
||||
assert_eq!(
|
||||
model["service_tiers"],
|
||||
serde_json::json!([{
|
||||
"id": "priority",
|
||||
"name": "Fast",
|
||||
"description": "Route the request through Codex fast mode."
|
||||
}])
|
||||
);
|
||||
assert_eq!(
|
||||
model["supported_reasoning_levels"],
|
||||
serde_json::json!([
|
||||
{
|
||||
"effort": "low",
|
||||
"description": "Fast responses with lighter reasoning"
|
||||
},
|
||||
{
|
||||
"effort": "medium",
|
||||
"description": "Balances speed and reasoning depth for everyday tasks"
|
||||
},
|
||||
{
|
||||
"effort": "high",
|
||||
"description": "Greater reasoning depth for complex problems"
|
||||
},
|
||||
{
|
||||
"effort": "xhigh",
|
||||
"description": "Extra high reasoning depth for complex problems"
|
||||
}
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_keep_the_codex_contract_when_profiles_are_empty() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models?client_version=0.145.0"))
|
||||
.await
|
||||
.expect("list empty Codex models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read empty Codex models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("Codex models JSON");
|
||||
|
||||
assert_eq!(value, serde_json::json!({ "models": [] }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_with_provider_profiles_should_keep_the_openai_list_contract() {
|
||||
let response = api_router(ModelsExecution::with_profiles())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models"))
|
||||
.await
|
||||
.expect("list compatible models response");
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read compatible models body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("models JSON");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
serde_json::json!({
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id":"model-a","object":"model","created":1700000000_i64,"owned_by":"gateway"},
|
||||
{"id":"model-b","object":"model","created":1700000000_i64,"owned_by":"gateway"}
|
||||
]
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_detail_should_keep_the_official_path_id_contract() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_core::operation::Operation;
|
||||
use serde_json::json;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, models::ModelsExecution};
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
@@ -141,3 +147,84 @@ fn response_create_should_reject_non_boolean_stream_without_disclosing_body_valu
|
||||
assert!(!rendered.contains(prompt));
|
||||
assert!(!rendered.contains(opaque_stream_value));
|
||||
}
|
||||
|
||||
// `serve_responses_websocket`/`forward_execution` 只有拿到 `hyper::upgrade::OnUpgrade`
|
||||
// 的真实 HTTP/1.1 升级连接才会运行;本 crate 的依赖图按 axum `ws` feature 编译
|
||||
// hyper(无 `http1`),测试进程内不存在能完成升级的服务端,`hyper::upgrade` 也不
|
||||
// 提供公开构造器。集成测试因此止步于 upgrade 边界:会话循环内的事件转发、错误帧
|
||||
// 与断连取消仍未被覆盖,需要 dev 依赖开启 `http1` 或 socket 泛化后才能落地。
|
||||
|
||||
const TEST_WEBSOCKET_KEY: &str = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
|
||||
fn upgrade_request(authorization: &str) -> Request<Body> {
|
||||
Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, authorization)
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build WebSocket upgrade request")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_responses_should_route_to_the_websocket_upgrade_boundary() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(upgrade_request("Bearer sk_models_test"))
|
||||
.await
|
||||
.expect("route WebSocket upgrade request");
|
||||
|
||||
// oneshot 请求不携带升级状态;426 证明 GET /v1/responses 进入的是
|
||||
// WebSocketUpgrade 边界而不是普通 HTTP handler。
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgradability_should_be_checked_before_authentication() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(upgrade_request("Bearer sk_invalid"))
|
||||
.await
|
||||
.expect("route unauthenticated upgrade request");
|
||||
|
||||
// 升级能力在 extractor 阶段先于 handler 内的 API Key 认证被校验,
|
||||
// 无效凭据得到的仍是升级失败而不是 401。
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgrade_should_reject_malformed_handshakes() {
|
||||
let missing_connection = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build request without connection header");
|
||||
let unsupported_version = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "12")
|
||||
.header("sec-websocket-key", TEST_WEBSOCKET_KEY)
|
||||
.body(Body::empty())
|
||||
.expect("build request with unsupported version");
|
||||
let missing_key = Request::get("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.header("connection", "upgrade")
|
||||
.header("upgrade", "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.body(Body::empty())
|
||||
.expect("build request without websocket key");
|
||||
|
||||
for request in [missing_connection, unsupported_version, missing_key] {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("route malformed WebSocket handshake");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,15 @@ use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
extract::DefaultBodyLimit,
|
||||
http::{Request, StatusCode},
|
||||
http::{Method, Request, StatusCode},
|
||||
routing::post,
|
||||
};
|
||||
use gateway_api::openai::router::MAX_CLIENT_REQUEST_BODY_BYTES;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
fn body_limit_app() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
@@ -50,3 +53,50 @@ async fn responses_body_limit_should_reject_one_byte_over_sixteen_mibibytes() {
|
||||
|
||||
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configured_cors_origin_assembles_router_and_answers_preflight() {
|
||||
let router = api_router_with_origins(
|
||||
ModelsExecution::new(),
|
||||
vec!["https://app.example.com".into()],
|
||||
)
|
||||
.await;
|
||||
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::OPTIONS)
|
||||
.uri("/v1/models")
|
||||
.header("origin", "https://app.example.com")
|
||||
.header("access-control-request-method", "GET")
|
||||
.header("access-control-request-headers", "authorization")
|
||||
.body(Body::empty())
|
||||
.expect("build preflight request"),
|
||||
)
|
||||
.await
|
||||
.expect("route preflight request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("https://app.example.com")
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("true")
|
||||
);
|
||||
let allow_headers = response
|
||||
.headers()
|
||||
.get("access-control-allow-headers")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("allow-headers present");
|
||||
assert!(allow_headers.contains("authorization"));
|
||||
assert!(allow_headers.contains("x-api-key"));
|
||||
assert!(allow_headers.contains("x-request-id"));
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
bytes = "1.11.1"
|
||||
chrono = "0.4.45"
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 唯一的账号重试、发送与下游 commit barrier owner。
|
||||
|
||||
use std::collections::{BTreeSet, VecDeque};
|
||||
use std::collections::BTreeSet;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -23,6 +23,7 @@ use crate::event::{
|
||||
};
|
||||
use crate::operation::{Operation, ProviderSessionState, RetrySafety};
|
||||
use crate::routing::RoutingPlan;
|
||||
use futures::future::Fuse;
|
||||
use futures::{FutureExt, StreamExt, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
|
||||
@@ -67,18 +68,18 @@ where
|
||||
let continuation_attempt =
|
||||
initial_continuation_attempt(&operation, &plan, continuation.as_ref());
|
||||
let image_generation_requested = operation.image_generation_requested();
|
||||
let commit_policy = if continuation_attempt == ContinuationAttempt::None {
|
||||
StreamCommitPolicy::FirstCanonicalEvent
|
||||
} else {
|
||||
StreamCommitPolicy::UntilOutputOrTerminal
|
||||
};
|
||||
|
||||
let mut session = ResponseExecutionSession {
|
||||
engine: Arc::clone(&self.engine),
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
timing_started_at,
|
||||
deadline,
|
||||
deadline_timer: Delay::new(
|
||||
deadline
|
||||
.duration_since(SystemTime::now())
|
||||
.unwrap_or(Duration::ZERO),
|
||||
)
|
||||
.fuse(),
|
||||
pending_request: Some(request),
|
||||
request_persisted: false,
|
||||
operation,
|
||||
@@ -86,13 +87,14 @@ where
|
||||
required_account,
|
||||
continuation,
|
||||
continuation_attempt,
|
||||
commit_policy,
|
||||
account_state_owner,
|
||||
cancellation,
|
||||
attempts: 0,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
current: None,
|
||||
send_state_watermark: UpstreamSendState::NotSent,
|
||||
downstream_committed_at: None,
|
||||
client_status_code: None,
|
||||
delivery_pending: false,
|
||||
@@ -106,7 +108,6 @@ where
|
||||
upstream_response_id: None,
|
||||
last_account_exhaustion: None,
|
||||
provider_attempt_outcomes: Vec::new(),
|
||||
before_commit: VecDeque::new(),
|
||||
};
|
||||
|
||||
if session.cancellation.is_cancelled() {
|
||||
@@ -141,12 +142,6 @@ struct FailureFinalization {
|
||||
provider_response_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum StreamCommitPolicy {
|
||||
FirstCanonicalEvent,
|
||||
UntilOutputOrTerminal,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话。
|
||||
///
|
||||
/// API 只能提交下游 delivery 边界;账号重试、断流终结与
|
||||
@@ -157,6 +152,8 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
client_api_key_ref: crate::policy::ClientApiKeyId,
|
||||
timing_started_at: Instant,
|
||||
deadline: SystemTime,
|
||||
/// 会话级 deadline 计时器;deadline 固定,帧循环内复用而非逐事件新建。
|
||||
deadline_timer: Fuse<Delay>,
|
||||
pending_request: Option<NewModelRequest>,
|
||||
request_persisted: bool,
|
||||
operation: Operation,
|
||||
@@ -164,13 +161,18 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
required_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
commit_policy: StreamCommitPolicy,
|
||||
account_state_owner: Option<ProviderAccountStateOwner>,
|
||||
cancellation: CancellationToken,
|
||||
attempts: u32,
|
||||
excluded_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算。
|
||||
/// 与 `required_account`(外部指定、贯穿整个请求)语义不同,不可合并。
|
||||
recovery_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
current: Option<CurrentAttempt>,
|
||||
/// 请求级发送状态水位;跨 attempt 单调不降,终态写回不得低于此档。
|
||||
send_state_watermark: UpstreamSendState,
|
||||
downstream_committed_at: Option<SystemTime>,
|
||||
client_status_code: Option<u16>,
|
||||
delivery_pending: bool,
|
||||
@@ -184,7 +186,6 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
upstream_response_id: Option<String>,
|
||||
last_account_exhaustion: Option<ProviderError>,
|
||||
provider_attempt_outcomes: Vec<ProviderAttemptOutcome>,
|
||||
before_commit: VecDeque<ProviderEvent>,
|
||||
}
|
||||
|
||||
impl<S: ?Sized> ResponseExecutionSession<S>
|
||||
@@ -203,15 +204,6 @@ where
|
||||
if self.finalized {
|
||||
return Ok(None);
|
||||
}
|
||||
if self.downstream_committed_at.is_some()
|
||||
&& let Some(event) = self.before_commit.pop_front()
|
||||
{
|
||||
return Ok(Some(CoordinatedEvent::single(
|
||||
event,
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)));
|
||||
}
|
||||
|
||||
loop {
|
||||
match self.pull().await? {
|
||||
PullOutcome::Event(event) => {
|
||||
@@ -221,25 +213,13 @@ where
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)));
|
||||
}
|
||||
let commit_significant = match self.commit_policy {
|
||||
StreamCommitPolicy::FirstCanonicalEvent => event.has_canonical_facts(),
|
||||
StreamCommitPolicy::UntilOutputOrTerminal => event.is_commit_significant(),
|
||||
};
|
||||
self.before_commit.push_back(event);
|
||||
if !commit_significant {
|
||||
continue;
|
||||
}
|
||||
self.delivery_pending = true;
|
||||
let events = self.before_commit.drain(..).collect();
|
||||
return CoordinatedEvent::try_batch(
|
||||
events,
|
||||
return Ok(Some(CoordinatedEvent::single(
|
||||
event,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.map(Some);
|
||||
}
|
||||
PullOutcome::AttemptDiscarded => {
|
||||
self.before_commit.clear();
|
||||
)));
|
||||
}
|
||||
PullOutcome::AttemptDiscarded => {}
|
||||
PullOutcome::End => {
|
||||
if self.downstream_committed_at.is_some() {
|
||||
self.finish_success().await?;
|
||||
@@ -406,11 +386,12 @@ where
|
||||
}
|
||||
|
||||
let boundary = {
|
||||
let current = self.current.as_mut().ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
poll_stream_item(
|
||||
&mut current.stream,
|
||||
self.cancellation.clone(),
|
||||
self.deadline,
|
||||
&mut self.deadline_timer,
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -421,7 +402,9 @@ where
|
||||
return Err(EngineError::Cancelled);
|
||||
}
|
||||
PollBoundary::Deadline => {
|
||||
self.record_current_provider_failure(ProviderErrorKind::Timeout);
|
||||
// 会话 deadline 是网关自身的请求预算,不是上游超时;
|
||||
// 真正的上游超时会作为流错误进入 `handle_stream_error` 记账。
|
||||
// 这里不写 provider 失败,避免长流集中到期误触 provider 熔断。
|
||||
self.finish_interruption(EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
@@ -498,6 +481,12 @@ where
|
||||
.checked_add(1)
|
||||
.and_then(NonZeroU32::new)
|
||||
.ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
// recovery 钉选在此被一次性消费,只绑定本次 replay attempt;
|
||||
// 外部 required_account 每次 attempt 都重新生效。
|
||||
let pinned_account = self
|
||||
.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.required_account.clone());
|
||||
let context = AttemptContext::new(
|
||||
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
|
||||
.with_timing_started_at(self.timing_started_at),
|
||||
@@ -506,15 +495,15 @@ where
|
||||
self.plan.account_selection_policy(),
|
||||
AccountAttemptContext::new(
|
||||
self.excluded_accounts.clone(),
|
||||
self.required_account.clone(),
|
||||
pinned_account.clone(),
|
||||
self.account_state_owner.clone(),
|
||||
)
|
||||
.with_credential_recovery_attempted(
|
||||
self.required_account.as_ref().is_some_and(|account| {
|
||||
.with_credential_recovery_attempted(pinned_account.as_ref().is_some_and(
|
||||
|account| {
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
}),
|
||||
),
|
||||
},
|
||||
)),
|
||||
self.continuation.clone(),
|
||||
self.cancellation.clone(),
|
||||
)
|
||||
@@ -547,10 +536,9 @@ where
|
||||
return Err(EngineError::Cancelled);
|
||||
}
|
||||
ProviderBoundary::Deadline => {
|
||||
self.record_provider_failure(
|
||||
candidate.provider().clone(),
|
||||
ProviderErrorKind::Timeout,
|
||||
);
|
||||
// 网关预算到期同样不是候选 Provider 的上游超时,不计入熔断;
|
||||
// Provider 自身的握手/传输超时会以 `ProviderErrorKind::Timeout`
|
||||
// 错误返回并在下方 `Result` 分支记账。
|
||||
self.finish_interruption(EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
@@ -597,8 +585,7 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
if self
|
||||
.required_account
|
||||
if pinned_account
|
||||
.as_ref()
|
||||
.is_some_and(|required| metadata.provider_account_id() != Some(required))
|
||||
{
|
||||
@@ -651,21 +638,30 @@ where
|
||||
account.clone(),
|
||||
));
|
||||
}
|
||||
self.persist_request().await?;
|
||||
self.engine
|
||||
.store()
|
||||
.record_attempt(AttemptRecord {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_count: next_attempt,
|
||||
trigger,
|
||||
provider_kind: metadata.provider().clone(),
|
||||
provider_account_id: metadata.provider_account_id().cloned(),
|
||||
provider_account_ref: metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: metadata.upstream_model().clone(),
|
||||
upstream_transport: metadata.transport().as_str().to_owned(),
|
||||
http_version: None,
|
||||
})
|
||||
.await?;
|
||||
let attempt_record = AttemptRecord {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_count: next_attempt,
|
||||
trigger,
|
||||
provider_kind: metadata.provider().clone(),
|
||||
provider_account_id: metadata.provider_account_id().cloned(),
|
||||
provider_account_ref: metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: metadata.upstream_model().clone(),
|
||||
upstream_transport: metadata.transport().as_str().to_owned(),
|
||||
http_version: None,
|
||||
};
|
||||
if self.request_persisted {
|
||||
self.engine.store().record_attempt(attempt_record).await?;
|
||||
} else {
|
||||
let request = self
|
||||
.pending_request
|
||||
.take()
|
||||
.ok_or(EngineError::InvalidDeliveryState)?;
|
||||
self.engine
|
||||
.store()
|
||||
.create_model_request_with_attempt(request, attempt_record)
|
||||
.await?;
|
||||
self.request_persisted = true;
|
||||
}
|
||||
self.attempts = next_attempt.get();
|
||||
self.current = Some(CurrentAttempt {
|
||||
stream,
|
||||
@@ -679,19 +675,6 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn persist_request(&mut self) -> Result<(), EngineError> {
|
||||
if self.request_persisted {
|
||||
return Ok(());
|
||||
}
|
||||
let request = self
|
||||
.pending_request
|
||||
.take()
|
||||
.ok_or(EngineError::InvalidDeliveryState)?;
|
||||
self.engine.store().create_model_request(request).await?;
|
||||
self.request_persisted = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) -> Result<(), EngineError> {
|
||||
let elapsed = elapsed_ms(self.timing_started_at);
|
||||
observe_event_timing(&mut self.timings, event, elapsed);
|
||||
@@ -706,25 +689,27 @@ where
|
||||
if let GatewayEvent::ProviderCost(observed) = event {
|
||||
self.cost = observed.into_estimate();
|
||||
}
|
||||
let current = self.current.as_mut().ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn observe_wire_event(&mut self) -> Result<(), EngineError> {
|
||||
let current = self.current.as_mut().ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -805,7 +790,7 @@ where
|
||||
|
||||
/// 返回 `true` 表示调用方必须丢弃本 attempt 已收集的未提交事件。
|
||||
async fn handle_stream_error(&mut self, error: ProviderError) -> Result<bool, EngineError> {
|
||||
let current = self.current.take().ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted
|
||||
@@ -813,11 +798,15 @@ where
|
||||
self.last_account_exhaustion = Some(error.clone());
|
||||
}
|
||||
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
|
||||
let send_state = if current.send_observed {
|
||||
// attempt_send_state 是本 attempt 自身的发送事实,驱动重试门;
|
||||
// 持久化与终态用请求级水位,二者不可混用(水位会把早先 attempt 的
|
||||
// sent 传染给本 attempt,从而错误放行/拦截重试)。
|
||||
let attempt_send_state = if current.send_observed {
|
||||
UpstreamSendState::Sent
|
||||
} else {
|
||||
error.send_state()
|
||||
};
|
||||
let send_state = self.raise_send_watermark(attempt_send_state);
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state)
|
||||
@@ -826,14 +815,14 @@ where
|
||||
let continuation_retry = self.prepare_continuation_retry(
|
||||
¤t,
|
||||
&error,
|
||||
send_state,
|
||||
attempt_send_state,
|
||||
provider_proved_replay_safe,
|
||||
);
|
||||
let ordinary_retry = self.required_account.is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& send_state != UpstreamSendState::Ambiguous
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& (self.operation.retry_safety() == RetrySafety::Idempotent
|
||||
|| provider_proved_replay_safe)
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
@@ -841,7 +830,7 @@ where
|
||||
&& provider_proved_replay_safe
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& send_state != UpstreamSendState::Ambiguous
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& self.attempts < self.plan.max_attempts().get()
|
||||
&& current
|
||||
.metadata
|
||||
@@ -858,7 +847,9 @@ where
|
||||
if let Some(account) = current.metadata.provider_account_id() {
|
||||
self.credential_recovery_attempted_accounts
|
||||
.insert(account.clone());
|
||||
self.required_account = Some(account.clone());
|
||||
// 只钉住紧随其后的 replay attempt;replay 再遇可重试错误时,
|
||||
// ordinary/continuation 重试门不受影响,仍可换号。
|
||||
self.recovery_account = Some(account.clone());
|
||||
}
|
||||
} else if !continuation_retry
|
||||
&& let Some(account) = current.metadata.provider_account_id()
|
||||
@@ -952,8 +943,8 @@ where
|
||||
self.timings.first_reasoning_ms = None;
|
||||
self.timings.first_text_ms = None;
|
||||
self.timings.first_token_ms = None;
|
||||
self.timings.provider_processing_ms = None;
|
||||
self.upstream_complete = false;
|
||||
self.before_commit.clear();
|
||||
}
|
||||
|
||||
async fn finish_success(&mut self) -> Result<(), EngineError> {
|
||||
@@ -982,8 +973,7 @@ where
|
||||
.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::status_code)
|
||||
.or(Some(200));
|
||||
.and_then(ProviderResponseObservation::status_code);
|
||||
let (upstream_transport, http_version, websocket_pool) =
|
||||
self.current_transport_observation();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
@@ -1028,6 +1018,7 @@ where
|
||||
error: &ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
) -> Result<(), EngineError> {
|
||||
let send_state = self.raise_send_watermark(send_state);
|
||||
let outcome = if error.kind() == ProviderErrorKind::Cancelled {
|
||||
ExecutionOutcome::Cancelled
|
||||
} else if self.downstream_committed_at.is_some() {
|
||||
@@ -1080,6 +1071,7 @@ where
|
||||
} else {
|
||||
UpstreamSendState::Ambiguous
|
||||
};
|
||||
let send_state = self.raise_send_watermark(send_state);
|
||||
if self.attempts > 0 {
|
||||
self.engine
|
||||
.store()
|
||||
@@ -1200,9 +1192,7 @@ where
|
||||
}
|
||||
|
||||
fn current_send_state(&self) -> UpstreamSendState {
|
||||
if self.attempts == 0 {
|
||||
UpstreamSendState::NotSent
|
||||
} else if self
|
||||
let observed = if self
|
||||
.current
|
||||
.as_ref()
|
||||
.is_some_and(|current| current.send_observed)
|
||||
@@ -1210,7 +1200,15 @@ where
|
||||
UpstreamSendState::Sent
|
||||
} else {
|
||||
UpstreamSendState::NotSent
|
||||
}
|
||||
};
|
||||
escalate_send_state(self.send_state_watermark, observed)
|
||||
}
|
||||
|
||||
/// 抬升并返回请求级发送水位;attempt 间切换(`current` 被取走)后,
|
||||
/// 后续终态沿用已达到的最高档,不会把已落库的 `sent` 写回 `not_sent`。
|
||||
fn raise_send_watermark(&mut self, observed: UpstreamSendState) -> UpstreamSendState {
|
||||
self.send_state_watermark = escalate_send_state(self.send_state_watermark, observed);
|
||||
self.send_state_watermark
|
||||
}
|
||||
|
||||
fn record_current_provider_success(&mut self) {
|
||||
@@ -1224,16 +1222,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn record_current_provider_failure(&mut self, error_kind: ProviderErrorKind) {
|
||||
let provider_kind = self
|
||||
.current
|
||||
.as_ref()
|
||||
.map(|current| current.metadata.provider().clone());
|
||||
if let Some(provider_kind) = provider_kind {
|
||||
self.record_provider_failure(provider_kind, error_kind);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_provider_failure(
|
||||
&mut self,
|
||||
provider_kind: crate::routing::ProviderKind,
|
||||
@@ -1263,17 +1251,17 @@ async fn poll_stream_item(
|
||||
stream: &mut ProviderStream,
|
||||
cancellation: CancellationToken,
|
||||
deadline: SystemTime,
|
||||
mut deadline_timer: &mut Fuse<Delay>,
|
||||
) -> PollBoundary {
|
||||
let Ok(remaining) = deadline.duration_since(SystemTime::now()) else {
|
||||
if SystemTime::now() >= deadline {
|
||||
return PollBoundary::Deadline;
|
||||
};
|
||||
}
|
||||
let next = stream.next().fuse();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let timeout = Delay::new(remaining).fuse();
|
||||
pin_mut!(next, cancelled, timeout);
|
||||
pin_mut!(next, cancelled);
|
||||
select_biased! {
|
||||
_ = cancelled => PollBoundary::Cancelled,
|
||||
_ = timeout => PollBoundary::Deadline,
|
||||
_ = deadline_timer => PollBoundary::Deadline,
|
||||
item = next => PollBoundary::Item(item),
|
||||
}
|
||||
}
|
||||
@@ -1338,6 +1326,18 @@ fn provider_proved_replay_safe(error: &ProviderError) -> bool {
|
||||
|| (error.send_state() != UpstreamSendState::Ambiguous && error.replay_is_safe())
|
||||
}
|
||||
|
||||
/// 发送状态合并档位:`Sent` > `Ambiguous` > `NotSent`。
|
||||
/// 只要任一 attempt 达到过高档,请求整体就不允许回落到低档。
|
||||
const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> UpstreamSendState {
|
||||
match (a, b) {
|
||||
(UpstreamSendState::Sent, _) | (_, UpstreamSendState::Sent) => UpstreamSendState::Sent,
|
||||
(UpstreamSendState::Ambiguous, _) | (_, UpstreamSendState::Ambiguous) => {
|
||||
UpstreamSendState::Ambiguous
|
||||
}
|
||||
(UpstreamSendState::NotSent, UpstreamSendState::NotSent) => UpstreamSendState::NotSent,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_event_timing(timings: &mut ModelRequestTimings, event: &GatewayEvent, elapsed_ms: u64) {
|
||||
timings.first_event_ms.get_or_insert(elapsed_ms);
|
||||
match event {
|
||||
|
||||
@@ -20,16 +20,20 @@ use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeResult};
|
||||
use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement, CoordinatedEvent,
|
||||
EngineError, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CoordinatedEvent, EngineError,
|
||||
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest, ProbeFailure, ProviderAccountId, ProviderAttemptOutcome, RecoveryReport,
|
||||
UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use crate::event::{GatewayEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use crate::operation::{Operation, ProviderSessionState};
|
||||
use crate::policy::{ClientApiKeyId, ClientPolicy};
|
||||
use crate::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use crate::routing::{ProviderKind, PublicModelId, RoutingContext, RuntimeSnapshot};
|
||||
use crate::routing::{
|
||||
ProviderKind, PublicModelId, PublicModelProfile, RoutingContext, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const CONTINUATION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
@@ -139,6 +143,9 @@ pub trait ExecutionService: Send + Sync {
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError>;
|
||||
fn public_models(&self, client: &AuthenticatedClient) -> Vec<PublicModelId>;
|
||||
fn public_model_profiles(&self, _client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
Vec::new()
|
||||
}
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool;
|
||||
fn start(
|
||||
&self,
|
||||
@@ -182,6 +189,8 @@ pub struct DefaultExecutionService {
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
probe_coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
/// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events。
|
||||
observations: Arc<dyn ExecutionStore>,
|
||||
providers: ProviderRegistry,
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
circuits: Arc<dyn ProviderCircuitPort>,
|
||||
@@ -200,6 +209,7 @@ impl DefaultExecutionService {
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
|
||||
) -> Self {
|
||||
let observations = Arc::clone(&execution);
|
||||
let engine = GatewayEngine::<dyn ExecutionStore>::new(execution, providers.clone());
|
||||
let transient: Arc<dyn ExecutionStore> = Arc::new(TransientExecutionStore);
|
||||
let probe_engine = GatewayEngine::<dyn ExecutionStore>::new(transient, providers.clone());
|
||||
@@ -207,6 +217,7 @@ impl DefaultExecutionService {
|
||||
snapshots,
|
||||
coordinator: Arc::new(AttemptCoordinator::new(engine)),
|
||||
probe_coordinator: Arc::new(AttemptCoordinator::new(probe_engine)),
|
||||
observations,
|
||||
providers,
|
||||
admissions,
|
||||
circuits,
|
||||
@@ -412,6 +423,11 @@ impl DefaultExecutionService {
|
||||
upstream_model,
|
||||
operation,
|
||||
} = request;
|
||||
let observed = ProbeObservation {
|
||||
provider_kind: provider_kind.clone(),
|
||||
account_id: account_id.clone(),
|
||||
upstream_model: upstream_model.clone(),
|
||||
};
|
||||
let snapshot = self.snapshots.acquire().map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -459,7 +475,7 @@ impl DefaultExecutionService {
|
||||
started_at,
|
||||
deadline_at,
|
||||
};
|
||||
let mut session = self
|
||||
let mut session = match self
|
||||
.probe_coordinator
|
||||
.start(
|
||||
new_request,
|
||||
@@ -470,18 +486,33 @@ impl DefaultExecutionService {
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
{
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
let events = session.collect_uncommitted().await;
|
||||
publish_provider_attempt_outcomes(
|
||||
self.circuits.as_ref(),
|
||||
session.provider_attempt_outcomes(),
|
||||
)
|
||||
.await;
|
||||
let events = events.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
session
|
||||
.commit_downstream(Some(200))
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
let events = match events {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(200)).await {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
Ok(AccountProbeResult {
|
||||
text: events
|
||||
.into_iter()
|
||||
@@ -493,6 +524,34 @@ impl DefaultExecutionService {
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> GatewayError {
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let _ = self
|
||||
.observations
|
||||
.record_probe_failure(ProbeFailure {
|
||||
provider_kind: observed.provider_kind.clone(),
|
||||
account_id: observed.account_id.clone(),
|
||||
upstream_model_id: observed.upstream_model.clone(),
|
||||
error: provider_error.clone(),
|
||||
latency: started_at.elapsed().unwrap_or_default(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
gateway_error_from_engine(error)
|
||||
}
|
||||
}
|
||||
|
||||
struct ProbeObservation {
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
upstream_model: UpstreamModelId,
|
||||
}
|
||||
|
||||
struct TransientExecutionStore;
|
||||
@@ -568,6 +627,12 @@ impl ExecutionService for DefaultExecutionService {
|
||||
.public_models_for_provider(client.policy.provider_kind())
|
||||
}
|
||||
|
||||
fn public_model_profiles(&self, client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
client
|
||||
.snapshot
|
||||
.public_model_profiles_for_provider(client.policy.provider_kind())
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
client
|
||||
.snapshot
|
||||
@@ -857,7 +922,7 @@ fn attach_continuation_session_state(operation: &mut Operation, pin: &NativeCont
|
||||
{
|
||||
return;
|
||||
}
|
||||
*operation = operation.clone().with_provider_session_state(state.clone());
|
||||
operation.set_provider_session_state(state.clone());
|
||||
}
|
||||
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
@@ -878,8 +943,3 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn commit_requirement(event: &CoordinatedEvent) -> CommitRequirement {
|
||||
event.commit_requirement()
|
||||
}
|
||||
|
||||
@@ -543,6 +543,16 @@ pub struct IntermediateFailure {
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// 不属于任何 `model_requests` 行的管理端账号探测失败。
|
||||
#[derive(Debug)]
|
||||
pub struct ProbeFailure {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub account_id: ProviderAccountId,
|
||||
pub upstream_model_id: UpstreamModelId,
|
||||
pub error: ProviderError,
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// `model_requests` 可用的毫秒级阶段耗时。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelRequestTimings {
|
||||
@@ -596,6 +606,19 @@ pub struct RecoveryReport {
|
||||
pub trait ExecutionStore: Send + Sync {
|
||||
async fn create_model_request(&self, request: NewModelRequest) -> Result<(), StoreError>;
|
||||
async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError>;
|
||||
/// 请求插入与首次 attempt 合并持久化;两者写同一行,支持合并写的
|
||||
/// store 可覆写为单次往返,缩短首 token 前的关键路径。
|
||||
///
|
||||
/// 覆写实现应保证原子性:默认的两步实现里第二步失败会留下一条
|
||||
/// `running` 请求行,由 deadline 回收器收敛。
|
||||
async fn create_model_request_with_attempt(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
attempt: AttemptRecord,
|
||||
) -> Result<(), StoreError> {
|
||||
self.create_model_request(request).await?;
|
||||
self.record_attempt(attempt).await
|
||||
}
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
@@ -616,6 +639,10 @@ pub trait ExecutionStore: Send + Sync {
|
||||
&self,
|
||||
failure: IntermediateFailure,
|
||||
) -> Result<(), StoreError>;
|
||||
/// 记录不挂在 `model_requests` 上的账号探测失败;默认丢弃。
|
||||
async fn record_probe_failure(&self, _failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: ModelRequestFinalization,
|
||||
@@ -646,11 +673,7 @@ impl CoordinatedEvent {
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建一个必须在同一下游提交边界内按序编码的事件批次。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空批次无法建立交付边界。
|
||||
/// 将一批 provider 事件合并为单个提交单元;空批次是无效投递状态。
|
||||
pub fn try_batch(
|
||||
events: Vec<ProviderEvent>,
|
||||
commit_requirement: CommitRequirement,
|
||||
@@ -701,6 +724,8 @@ pub enum EngineError {
|
||||
Deadline,
|
||||
#[error("routing plan has no candidate")]
|
||||
EmptyRoutingPlan,
|
||||
#[error("no provider attempt is active")]
|
||||
NoActiveAttempt,
|
||||
#[error("downstream delivery must be committed before execution can continue")]
|
||||
DownstreamCommitRequired,
|
||||
#[error("downstream delivery cannot be committed in the current state")]
|
||||
|
||||
@@ -20,7 +20,9 @@ use crate::error::{
|
||||
};
|
||||
use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::operation::Operation;
|
||||
use crate::routing::{ModelCapabilities, ProviderCandidate, ProviderKind, UpstreamModelId};
|
||||
use crate::routing::{
|
||||
ModelCapabilities, ModelPresentation, ProviderCandidate, ProviderKind, UpstreamModelId,
|
||||
};
|
||||
|
||||
/// Box 只出现在 Provider Registry 的统一 event envelope 边界。
|
||||
pub type EventStream =
|
||||
@@ -433,6 +435,7 @@ pub struct ProviderRequestObservation {
|
||||
pub struct ProviderModelCapabilities {
|
||||
upstream_model: UpstreamModelId,
|
||||
capabilities: ModelCapabilities,
|
||||
presentation: Option<ModelPresentation>,
|
||||
}
|
||||
|
||||
/// Provider 实时目录成功发布后的进程内单调代次。
|
||||
@@ -459,9 +462,16 @@ impl ProviderModelCapabilities {
|
||||
Self {
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_presentation(mut self, presentation: ModelPresentation) -> Self {
|
||||
self.presentation = Some(presentation);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
@@ -471,6 +481,11 @@ impl ProviderModelCapabilities {
|
||||
pub const fn capabilities(&self) -> &ModelCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> Option<&ModelPresentation> {
|
||||
self.presentation.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderRequest {
|
||||
|
||||
@@ -715,12 +715,6 @@ impl ProviderEvent {
|
||||
&self.canonical
|
||||
}
|
||||
|
||||
/// 返回 Core 可改写客户端响应身份的全部 facts。
|
||||
#[must_use]
|
||||
pub fn canonical_facts_mut(&mut self) -> &mut [GatewayEvent] {
|
||||
&mut self.canonical
|
||||
}
|
||||
|
||||
/// 返回协议原生表达。
|
||||
#[must_use]
|
||||
pub fn wire_event(&self) -> Option<&ProtocolWireEvent> {
|
||||
@@ -756,18 +750,6 @@ impl ProviderEvent {
|
||||
pub fn has_client_event(&self) -> bool {
|
||||
!self.canonical.is_empty() || self.wire.is_some()
|
||||
}
|
||||
|
||||
/// 返回是否包含足以冻结下游 commit barrier 的 canonical fact。
|
||||
///
|
||||
/// 单独的 wire event 不能冻结边界:协议层在看到 `Started`
|
||||
/// 携带的网关响应身份前无法安全编码它。Core 会保留这些
|
||||
/// 事件,直到首个 canonical 可见增量或终态再一并交付。
|
||||
#[must_use]
|
||||
pub fn is_commit_significant(&self) -> bool {
|
||||
self.canonical
|
||||
.iter()
|
||||
.any(GatewayEvent::is_commit_significant)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayEvent> for ProviderEvent {
|
||||
@@ -776,23 +758,6 @@ impl From<GatewayEvent> for ProviderEvent {
|
||||
}
|
||||
}
|
||||
|
||||
impl GatewayEvent {
|
||||
/// 返回该 fact 是否足以冻结下游 commit barrier。
|
||||
///
|
||||
/// 生命周期、结构与结算 facts 可以在换号前安全丢弃;首个可见增量或
|
||||
/// Provider 正常终态才会使客户端输出不可撤回。
|
||||
#[must_use]
|
||||
pub const fn is_commit_significant(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::TextDelta(_)
|
||||
| Self::ReasoningDelta(_)
|
||||
| Self::ToolCallDelta(_)
|
||||
| Self::Completed(_)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical event 顺序错误。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum EventSequenceError {
|
||||
|
||||
@@ -269,10 +269,15 @@ impl GenerateRequest {
|
||||
/// 附着同一客户端连接上一轮由 Provider 返回的不透明状态。
|
||||
#[must_use]
|
||||
pub fn with_provider_session_state(mut self, state: ProviderSessionState) -> Self {
|
||||
Arc::make_mut(&mut self.payload).provider_session_state = Some(state);
|
||||
self.set_provider_session_state(state);
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地附着会话状态;payload 独占时不复制正文。
|
||||
pub fn set_provider_session_state(&mut self, state: ProviderSessionState) {
|
||||
Arc::make_mut(&mut self.payload).provider_session_state = Some(state);
|
||||
}
|
||||
|
||||
/// 返回最大输出 token 数。
|
||||
#[must_use]
|
||||
pub fn max_output_tokens(&self) -> Option<u64> {
|
||||
@@ -597,6 +602,13 @@ impl Operation {
|
||||
}
|
||||
}
|
||||
|
||||
/// 原地附着 Provider 私有状态;payload 独占时不复制正文。
|
||||
pub fn set_provider_session_state(&mut self, state: ProviderSessionState) {
|
||||
if let Self::Generate(request) = self {
|
||||
request.set_provider_session_state(state);
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回稳定 operation 分类。
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> OperationKind {
|
||||
|
||||
@@ -474,9 +474,22 @@ pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次瞬态刷新失败并返回窗口内累计失败次数;每次失败刷新 TTL。
|
||||
fn record_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 凭据完整成功轮换后清零失败计数,退避窗口重新从 base 起步。
|
||||
fn clear_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// 临时 cooldown 只保存调度截止时间;原因事实仍由 PostgreSQL 账号状态持有。
|
||||
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
@@ -592,6 +605,33 @@ impl ProviderRefreshPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// 指数退避基准延迟;attempt=1 即为该值。
|
||||
const REFRESH_BACKOFF_BASE_DELAY: Duration = Duration::from_secs(5);
|
||||
/// 每多一次连续失败,基准延迟乘以该因子。
|
||||
const REFRESH_BACKOFF_FACTOR: u32 = 3;
|
||||
/// 退避延迟上限,避免连续失败时无限增长。
|
||||
const REFRESH_BACKOFF_CAP: Duration = Duration::from_secs(300);
|
||||
|
||||
/// 基于连续失败计数的指数退避重试时刻;复用 `provider_refresh_retry_at` 的稳定扰动。
|
||||
///
|
||||
/// `attempt` 为窗口内累计失败次数(0 与 1 等价,均取基准延迟)。延迟按
|
||||
/// `base * factor^(attempt-1)` 增长并封顶到 `REFRESH_BACKOFF_CAP`。
|
||||
pub fn provider_refresh_backoff_at(
|
||||
account_id: &ProviderAccountId,
|
||||
observed_at: SystemTime,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let exponent = attempt.saturating_sub(1);
|
||||
let multiplier = REFRESH_BACKOFF_FACTOR.saturating_pow(exponent);
|
||||
let scaled_seconds = REFRESH_BACKOFF_BASE_DELAY
|
||||
.as_secs()
|
||||
.saturating_mul(u64::from(multiplier))
|
||||
.min(REFRESH_BACKOFF_CAP.as_secs());
|
||||
let base_delay = Duration::from_secs(scaled_seconds);
|
||||
provider_refresh_retry_at(account_id, observed_at, base_delay, reason)
|
||||
}
|
||||
|
||||
/// 临时失败后的持久重试时刻;稳定扰动避免多实例同频重试。
|
||||
pub fn provider_refresh_retry_at(
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -136,6 +136,240 @@ pub struct ModelCapabilities {
|
||||
upstream_validates_features: bool,
|
||||
}
|
||||
|
||||
/// Provider 为客户端模型目录提供的展示与交互能力。
|
||||
///
|
||||
/// 该值不参与路由,也不把任一 Provider 的 wire 类型带入 Core。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelPresentation {
|
||||
display_name: Option<String>,
|
||||
description: Option<String>,
|
||||
default_reasoning_effort: Option<String>,
|
||||
supported_reasoning_efforts: Vec<String>,
|
||||
context_window_tokens: Option<u64>,
|
||||
image_input: bool,
|
||||
agent_tools: bool,
|
||||
parallel_tool_calls: bool,
|
||||
search_tool: bool,
|
||||
image_detail_original: bool,
|
||||
verbosity: bool,
|
||||
service_tiers: Vec<ModelServiceTier>,
|
||||
hidden: bool,
|
||||
}
|
||||
|
||||
/// Provider 声明给 Codex 客户端的服务档位。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ModelServiceTier {
|
||||
id: String,
|
||||
name: String,
|
||||
description: String,
|
||||
speed_tier: Option<String>,
|
||||
}
|
||||
|
||||
impl ModelServiceTier {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
id: impl Into<String>,
|
||||
name: impl Into<String>,
|
||||
description: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
speed_tier: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_speed_tier(mut self, speed_tier: impl Into<String>) -> Self {
|
||||
self.speed_tier = Some(speed_tier.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
&self.id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn description(&self) -> &str {
|
||||
&self.description
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn speed_tier(&self) -> Option<&str> {
|
||||
self.speed_tier.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelPresentation {
|
||||
#[must_use]
|
||||
pub fn new(display_name: Option<String>, description: Option<String>) -> Self {
|
||||
Self {
|
||||
display_name,
|
||||
description,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_reasoning(
|
||||
mut self,
|
||||
default_effort: Option<String>,
|
||||
supported_efforts: Vec<String>,
|
||||
) -> Self {
|
||||
self.default_reasoning_effort = default_effort;
|
||||
self.supported_reasoning_efforts = supported_efforts;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_context_window_tokens(mut self, context_window_tokens: Option<u64>) -> Self {
|
||||
self.context_window_tokens = context_window_tokens;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_image_input(mut self, image_input: bool) -> Self {
|
||||
self.image_input = image_input;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_agent_tools(mut self, agent_tools: bool, parallel_tool_calls: bool) -> Self {
|
||||
self.agent_tools = agent_tools;
|
||||
self.parallel_tool_calls = parallel_tool_calls;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_search_tool(mut self, search_tool: bool) -> Self {
|
||||
self.search_tool = search_tool;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_image_detail_original(mut self, image_detail_original: bool) -> Self {
|
||||
self.image_detail_original = image_detail_original;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_verbosity(mut self, verbosity: bool) -> Self {
|
||||
self.verbosity = verbosity;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_service_tiers(mut self, service_tiers: Vec<ModelServiceTier>) -> Self {
|
||||
self.service_tiers = service_tiers;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_hidden(mut self, hidden: bool) -> Self {
|
||||
self.hidden = hidden;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn display_name(&self) -> Option<&str> {
|
||||
self.display_name.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn description(&self) -> Option<&str> {
|
||||
self.description.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn default_reasoning_effort(&self) -> Option<&str> {
|
||||
self.default_reasoning_effort.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn supported_reasoning_efforts(&self) -> &[String] {
|
||||
&self.supported_reasoning_efforts
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn context_window_tokens(&self) -> Option<u64> {
|
||||
self.context_window_tokens
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn image_input(&self) -> bool {
|
||||
self.image_input
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn agent_tools(&self) -> bool {
|
||||
self.agent_tools
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn parallel_tool_calls(&self) -> bool {
|
||||
self.parallel_tool_calls
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn search_tool(&self) -> bool {
|
||||
self.search_tool
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn image_detail_original(&self) -> bool {
|
||||
self.image_detail_original
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn verbosity(&self) -> bool {
|
||||
self.verbosity
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn service_tiers(&self) -> &[ModelServiceTier] {
|
||||
&self.service_tiers
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn hidden(&self) -> bool {
|
||||
self.hidden
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个公开模型及其 Provider 编译后的客户端画像。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PublicModelProfile {
|
||||
model: PublicModelId,
|
||||
presentation: ModelPresentation,
|
||||
}
|
||||
|
||||
impl PublicModelProfile {
|
||||
#[must_use]
|
||||
pub const fn new(model: PublicModelId, presentation: ModelPresentation) -> Self {
|
||||
Self {
|
||||
model,
|
||||
presentation,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn model(&self) -> &PublicModelId {
|
||||
&self.model
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> &ModelPresentation {
|
||||
&self.presentation
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelCapabilities {
|
||||
#[must_use]
|
||||
pub fn new(operations: BTreeSet<OperationKind>, max_output_tokens: Option<u64>) -> Self {
|
||||
@@ -205,6 +439,7 @@ pub struct ProviderModel {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
capabilities: ModelCapabilities,
|
||||
presentation: Option<ModelPresentation>,
|
||||
}
|
||||
|
||||
impl ProviderModel {
|
||||
@@ -218,9 +453,16 @@ impl ProviderModel {
|
||||
provider,
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_presentation(mut self, presentation: ModelPresentation) -> Self {
|
||||
self.presentation = Some(presentation);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider(&self) -> &ProviderKind {
|
||||
&self.provider
|
||||
@@ -230,6 +472,11 @@ impl ProviderModel {
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn presentation(&self) -> Option<&ModelPresentation> {
|
||||
self.presentation.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// 本次请求选择 Provider 时使用的动态过滤事实。
|
||||
|
||||
@@ -231,11 +231,15 @@ async fn compile_runtime_snapshot(
|
||||
continue;
|
||||
};
|
||||
provider_models.extend(models.into_iter().map(|model| {
|
||||
ProviderModel::new(
|
||||
let compiled = ProviderModel::new(
|
||||
provider.clone(),
|
||||
model.upstream_model().clone(),
|
||||
model.capabilities().clone(),
|
||||
)
|
||||
);
|
||||
match model.presentation().cloned() {
|
||||
Some(presentation) => compiled.with_presentation(presentation),
|
||||
None => compiled,
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -287,6 +291,8 @@ pub struct RuntimeSnapshot {
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
providers: Arc<BTreeSet<ProviderKind>>,
|
||||
provider_models: Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>>,
|
||||
provider_model_presentations:
|
||||
Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>>,
|
||||
model_mappings: Arc<BTreeMap<String, String>>,
|
||||
provider_catalog_generations: Arc<BTreeMap<ProviderKind, ProviderCatalogGeneration>>,
|
||||
client_policies: Arc<BTreeMap<ClientApiKeyId, ClientPolicy>>,
|
||||
@@ -313,17 +319,24 @@ impl RuntimeSnapshot {
|
||||
|
||||
let mut model_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>::new();
|
||||
let mut presentation_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>::new();
|
||||
for model in provider_models {
|
||||
if !provider_set.contains(model.provider()) {
|
||||
let ProviderModel {
|
||||
provider,
|
||||
upstream_model,
|
||||
capabilities,
|
||||
presentation,
|
||||
} = model;
|
||||
if !provider_set.contains(&provider) {
|
||||
return Err(RoutingError::NotFound {
|
||||
entity: "provider",
|
||||
id: model.provider().to_string(),
|
||||
id: provider.to_string(),
|
||||
});
|
||||
}
|
||||
let models = model_map.entry(model.provider).or_default();
|
||||
let upstream_model = model.upstream_model;
|
||||
let models = model_map.entry(provider.clone()).or_default();
|
||||
if models
|
||||
.insert(upstream_model.clone(), model.capabilities)
|
||||
.insert(upstream_model.clone(), capabilities)
|
||||
.is_some()
|
||||
{
|
||||
return Err(RoutingError::DuplicateEntity {
|
||||
@@ -331,6 +344,12 @@ impl RuntimeSnapshot {
|
||||
id: upstream_model.to_string(),
|
||||
});
|
||||
}
|
||||
if let Some(presentation) = presentation {
|
||||
presentation_map
|
||||
.entry(provider)
|
||||
.or_default()
|
||||
.insert(upstream_model, presentation);
|
||||
}
|
||||
}
|
||||
|
||||
let mut client_policy_map = BTreeMap::new();
|
||||
@@ -350,6 +369,7 @@ impl RuntimeSnapshot {
|
||||
account_selection_policy,
|
||||
providers: Arc::new(provider_set),
|
||||
provider_models: Arc::new(model_map),
|
||||
provider_model_presentations: Arc::new(presentation_map),
|
||||
model_mappings: Arc::new(BTreeMap::new()),
|
||||
provider_catalog_generations: Arc::new(BTreeMap::new()),
|
||||
client_policies: Arc::new(client_policy_map),
|
||||
@@ -402,6 +422,38 @@ impl RuntimeSnapshot {
|
||||
models.into_iter().collect()
|
||||
}
|
||||
|
||||
/// 返回 Provider 已明确声明画像的公开模型;没有画像时不猜测 Provider 语义。
|
||||
#[must_use]
|
||||
pub fn public_model_profiles_for_provider(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> Vec<super::PublicModelProfile> {
|
||||
let Some(presentations) = self.provider_model_presentations.get(provider) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut profiles = BTreeMap::new();
|
||||
for (model, presentation) in presentations {
|
||||
if let Ok(public_model) = PublicModelId::new(model.as_str().to_owned()) {
|
||||
profiles.insert(public_model, presentation.clone());
|
||||
}
|
||||
}
|
||||
for alias in self.model_mappings.keys() {
|
||||
let target = self.mapped_model(alias);
|
||||
let Some(presentation) = presentations.iter().find_map(|(model, presentation)| {
|
||||
(model.as_str() == target).then_some(presentation)
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
if let Ok(public_model) = PublicModelId::new(alias.clone()) {
|
||||
profiles.insert(public_model, presentation.clone());
|
||||
}
|
||||
}
|
||||
profiles
|
||||
.into_iter()
|
||||
.map(|(model, presentation)| super::PublicModelProfile::new(model, presentation))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn public_models(&self) -> Vec<PublicModelId> {
|
||||
self.providers
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::StreamExt;
|
||||
use futures::executor::block_on;
|
||||
|
||||
use gateway_core::accounting::{CalculatedCost, CostSource, ProviderReportedCost, Usage};
|
||||
@@ -15,18 +16,20 @@ use gateway_core::engine::continuation::{
|
||||
use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, CredentialRevision, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCallMetadata, ProviderCatalogGeneration, ProviderModelCapabilities,
|
||||
ProviderRegistry, ProviderRequest, ProviderResource, ProviderStream, UpstreamTransport,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement,
|
||||
ContinuationAttempt, ExecutionOutcome, ExecutionStore, GatewayEngine, IntermediateFailure,
|
||||
ModelRequestFinalization, ModelRequestId, NewModelRequest, ProviderAttemptOutcome,
|
||||
RecoveryReport, UpstreamSendState,
|
||||
ContinuationAttempt, EngineError, ExecutionOutcome, ExecutionStore, GatewayEngine,
|
||||
IntermediateFailure, ModelRequestFinalization, ModelRequestId, NewModelRequest,
|
||||
ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ContinuationFailure, ProviderError, ProviderErrorKind, SafeUpstreamValue, StoreError,
|
||||
ContinuationFailure, GatewayErrorKind, ProviderError, ProviderErrorKind, SafeUpstreamValue,
|
||||
StoreError,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -221,6 +224,11 @@ enum Script {
|
||||
account_id: &'static str,
|
||||
items: Vec<Result<GatewayEvent, ProviderError>>,
|
||||
},
|
||||
/// 产出 `items` 后永久悬挂的流;用于逼出会话级 deadline。
|
||||
HangingStream {
|
||||
account_id: &'static str,
|
||||
items: Vec<Result<GatewayEvent, ProviderError>>,
|
||||
},
|
||||
ObservedStream {
|
||||
account_id: &'static str,
|
||||
items: Vec<Result<ProviderEvent, ProviderError>>,
|
||||
@@ -296,6 +304,30 @@ impl Provider for ScriptedProvider {
|
||||
(),
|
||||
))
|
||||
}
|
||||
Script::HangingStream { account_id, items } => {
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
candidate.upstream_model().clone(),
|
||||
ProviderResource::Account {
|
||||
id: ProviderAccountId::new(account_id).expect("account id"),
|
||||
revision: CredentialRevision::new(1).expect("revision"),
|
||||
},
|
||||
UpstreamTransport::new("http_sse").expect("transport"),
|
||||
);
|
||||
Ok(ProviderStream::new(
|
||||
metadata,
|
||||
Box::pin(
|
||||
futures::stream::iter(
|
||||
items
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::from)),
|
||||
)
|
||||
.chain(futures::stream::pending()),
|
||||
),
|
||||
(),
|
||||
))
|
||||
}
|
||||
Script::ObservedStream { account_id, items } => {
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
@@ -855,6 +887,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
first_event_ms: Some(987_654),
|
||||
provider_processing_ms: Some(41),
|
||||
..ProviderResponseTimings::default()
|
||||
});
|
||||
let second_observation = ProviderResponseObservation::new(
|
||||
@@ -910,6 +943,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
assert_eq!(finalization.transport_decision_wait_ms, None);
|
||||
assert_eq!(finalization.connect_ms, None);
|
||||
assert_eq!(finalization.headers_ms, None);
|
||||
assert_eq!(finalization.provider_processing_ms, None);
|
||||
assert!(
|
||||
finalization
|
||||
.first_event_ms
|
||||
@@ -948,6 +982,44 @@ fn websocket_success_keeps_client_http_status_absent() {
|
||||
assert_eq!(state.finalizations[0].client_status_code, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_without_response_observation_persists_no_upstream_status() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: complete_stream(None),
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
block_on(session.collect_uncommitted()).expect("collect response");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(state.finalizations[0].upstream_status_code, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_attempt_is_classified_internal_not_no_available_provider() {
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&EngineError::EmptyRoutingPlan).kind(),
|
||||
GatewayErrorKind::NoAvailableProvider
|
||||
);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&EngineError::NoActiveAttempt).kind(),
|
||||
GatewayErrorKind::Internal
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_native_continuation_reaches_every_attempt_context() {
|
||||
let operation = generate_operation();
|
||||
@@ -1005,6 +1077,51 @@ fn authenticated_native_continuation_reaches_every_attempt_context() {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_native_continuation_delivers_started_event_before_later_output() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, _, _) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: complete_stream(None),
|
||||
}]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response").expect("previous response"),
|
||||
SafeUpstreamValue::new("upstream-response").expect("upstream response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let first = block_on(session.next_event())
|
||||
.expect("first event")
|
||||
.expect("started event");
|
||||
assert_eq!(
|
||||
first.commit_requirement(),
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
);
|
||||
let events = first.into_provider_events();
|
||||
assert!(matches!(
|
||||
events.as_slice(),
|
||||
[event]
|
||||
if matches!(event.canonical_facts(), [GatewayEvent::Started(_)])
|
||||
));
|
||||
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit started event");
|
||||
while block_on(session.next_event())
|
||||
.expect("remaining event")
|
||||
.is_some()
|
||||
{}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
@@ -1479,6 +1596,62 @@ fn recovered_credential_retries_the_same_account_exactly_once() {
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retryable_error_after_credential_recovery_switches_account_instead_of_terminating() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Unauthorized,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(401)
|
||||
.with_replay_safe()
|
||||
.with_same_account_retry())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("third attempt succeeds on another account");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
let first = ProviderAccountId::new("acct_first").expect("account");
|
||||
assert_eq!(contexts.len(), 3);
|
||||
assert_eq!(contexts[1].required_account(), Some(&first));
|
||||
assert!(contexts[1].credential_recovery_attempted());
|
||||
// recovery 钉账号只绑定 replay attempt;replay 上的 429 之后必须能换号。
|
||||
assert_eq!(contexts[2].required_account(), None);
|
||||
assert!(contexts[2].excluded_accounts().contains(&first));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 3);
|
||||
assert_eq!(state.intermediate_failures, 2);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_idempotent_explicit_429_rejection_rotates_account_before_output() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
@@ -1575,6 +1748,8 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations.len(), 1);
|
||||
// attempt-1 已把 sent 落库;attempt-2 空选路终态不得降级回 not_sent。
|
||||
assert_eq!(state.finalizations[0].send_state, UpstreamSendState::Sent);
|
||||
assert_eq!(state.finalizations[0].upstream_status_code, Some(429));
|
||||
assert_eq!(
|
||||
state.finalizations[0].provider_error_code.as_deref(),
|
||||
@@ -1906,3 +2081,71 @@ fn expired_deadline_finalizes_without_calling_provider() {
|
||||
assert!(state.attempts.is_empty());
|
||||
assert!(state.finalizations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadline_after_commit_is_incomplete_without_provider_circuit_failure() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::HangingStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Ok(GatewayEvent::Started(ResponseMeta::new(
|
||||
"response-1",
|
||||
"gpt-5",
|
||||
)))],
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_millis(300)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let first = block_on(session.next_event())
|
||||
.expect("first event")
|
||||
.expect("started event");
|
||||
assert_eq!(
|
||||
first.commit_requirement(),
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit started event");
|
||||
|
||||
let error = block_on(session.next_event()).expect_err("deadline elapses mid-stream");
|
||||
|
||||
assert!(matches!(error, EngineError::Deadline));
|
||||
// 网关自身请求预算到期不是上游超时;已在交付中的长流集中到期
|
||||
// 不得作为 provider Timeout 计入熔断。
|
||||
assert!(session.provider_attempt_outcomes().is_empty());
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Incomplete);
|
||||
assert!(state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadline_before_first_event_records_no_provider_circuit_failure() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::HangingStream {
|
||||
account_id: "acct_first",
|
||||
items: Vec::new(),
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_millis(300)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let error = block_on(session.next_event()).expect_err("deadline elapses before first event");
|
||||
|
||||
assert!(matches!(error, EngineError::Deadline));
|
||||
assert!(session.provider_attempt_outcomes().is_empty());
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert!(!state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
@@ -24,12 +24,15 @@ use gateway_core::engine::execution::{
|
||||
ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit,
|
||||
};
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::ProviderRegistry;
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest, RecoveryReport, UpstreamSendState,
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
|
||||
ProviderRequest, ProviderStream,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
@@ -78,6 +81,64 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() {
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_failures_should_be_observable_without_a_model_request_row() {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry");
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(probe_snapshot()),
|
||||
store.clone(),
|
||||
providers,
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
|
||||
let error = block_on(service.probe(AccountProbeRequest {
|
||||
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
|
||||
operation: probe_operation(),
|
||||
}))
|
||||
.expect_err("the provider rejects every probe");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
|
||||
}
|
||||
|
||||
struct FailingProvider;
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for FailingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
_request: ProviderRequest,
|
||||
_context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_authentication_should_record_client_key_usage() {
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
@@ -121,12 +182,20 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
|
||||
#[derive(Default)]
|
||||
struct TrackingExecutionStore {
|
||||
touched: AtomicBool,
|
||||
probe_failures: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl TrackingExecutionStore {
|
||||
fn touch(&self) {
|
||||
self.touched.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn probe_failures(&self) -> Vec<String> {
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -170,6 +239,14 @@ impl ExecutionStore for TrackingExecutionStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.push(failure.error.kind().as_str().to_owned());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
|
||||
self.touch();
|
||||
Ok(())
|
||||
|
||||
@@ -24,8 +24,8 @@ use gateway_core::routing::snapshot::{
|
||||
SnapshotSubscriptionError, SnapshotSubscriptionPort, runtime_revision_needs_refresh,
|
||||
};
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, PublicModelId, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
ConfigRevision, ModelCapabilities, ModelPresentation, ProviderKind, PublicModelId,
|
||||
RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
use gateway_core::task::WorkerKind;
|
||||
|
||||
@@ -135,13 +135,19 @@ impl Provider for PublishingCatalogProvider {
|
||||
if self.queries.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
self.generation.store(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(vec![ProviderModelCapabilities::new(
|
||||
UpstreamModelId::new("upstream-model").expect("model"),
|
||||
ModelCapabilities::new(
|
||||
std::collections::BTreeSet::from([OperationKind::Generate]),
|
||||
Ok(vec![
|
||||
ProviderModelCapabilities::new(
|
||||
UpstreamModelId::new("upstream-model").expect("model"),
|
||||
ModelCapabilities::new(
|
||||
std::collections::BTreeSet::from([OperationKind::Generate]),
|
||||
None,
|
||||
),
|
||||
)
|
||||
.with_presentation(ModelPresentation::new(
|
||||
Some("Upstream Model".to_owned()),
|
||||
None,
|
||||
),
|
||||
)])
|
||||
)),
|
||||
])
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
@@ -215,6 +221,15 @@ fn compiler_retries_when_provider_publishes_catalog_during_compilation() {
|
||||
.map(|generation| generation.get()),
|
||||
Some(1),
|
||||
);
|
||||
let profiles =
|
||||
snapshot.public_model_profiles_for_provider(&ProviderKind::new("alpha").expect("provider"));
|
||||
assert_eq!(
|
||||
profiles
|
||||
.iter()
|
||||
.map(|profile| profile.model().as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["public-model", "upstream-model"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -28,7 +28,6 @@ tokio = { version = "1.52.3", default-features = false, features = ["macros", "r
|
||||
tracing = "0.1.44"
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
filetime = "0.2.27"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
pub mod config;
|
||||
mod logging;
|
||||
mod serve;
|
||||
pub mod serve;
|
||||
pub mod system_update;
|
||||
pub mod workers;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! HTTP 监听、OS signal、原子连接注册与优雅 drain。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
@@ -12,7 +12,12 @@ use tokio::sync::Notify;
|
||||
const DRAINING_BIT: usize = 1usize << (usize::BITS - 1);
|
||||
const ACTIVE_MASK: usize = !DRAINING_BIT;
|
||||
|
||||
pub(crate) struct ConnectionTracker {
|
||||
/// 自重启交接没有显式握手:替换进程按 CPR_RESTART_DELAY_MS 估算等待,
|
||||
/// 可能在旧进程释放监听端口前尝试绑定,因此对 AddrInUse 保留有限重试窗口。
|
||||
const BIND_RETRY_WINDOW: Duration = Duration::from_secs(10);
|
||||
const BIND_RETRY_MAX_DELAY: Duration = Duration::from_secs(1);
|
||||
|
||||
pub struct ConnectionTracker {
|
||||
state: Arc<ConnectionState>,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
@@ -23,7 +28,8 @@ struct ConnectionState {
|
||||
}
|
||||
|
||||
impl ConnectionTracker {
|
||||
pub(crate) fn new(cancellation: CancellationToken) -> Self {
|
||||
#[must_use]
|
||||
pub fn new(cancellation: CancellationToken) -> Self {
|
||||
Self {
|
||||
state: Arc::new(ConnectionState {
|
||||
value: AtomicUsize::new(0),
|
||||
@@ -41,10 +47,15 @@ impl ConnectionTracker {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_idle(&self, timeout: Duration) {
|
||||
pub async fn wait_until_idle(&self, timeout: Duration) {
|
||||
let wait = async {
|
||||
loop {
|
||||
let notified = self.state.idle.notified();
|
||||
tokio::pin!(notified);
|
||||
// notify_waiters 不保存许可:必须先 enable 注册等待者,再读
|
||||
// 活跃计数;否则最后一个 guard 在读数与首次 poll 之间 drop
|
||||
// 时唤醒丢失,只能等满整个超时。
|
||||
notified.as_mut().enable();
|
||||
if self.state.value.load(Ordering::Acquire) & ACTIVE_MASK == 0 {
|
||||
return;
|
||||
}
|
||||
@@ -102,6 +113,27 @@ impl Drop for ActiveConnection {
|
||||
}
|
||||
}
|
||||
|
||||
/// 绑定失败即进程退出、服务彻底离线,因此 AddrInUse(典型为自重启交接
|
||||
/// 时旧进程尚未关闭 listener)在窗口内指数退避重试;其他错误立即上抛。
|
||||
pub async fn bind_listener(address: &str) -> std::io::Result<tokio::net::TcpListener> {
|
||||
let deadline = tokio::time::Instant::now() + BIND_RETRY_WINDOW;
|
||||
let mut delay = Duration::from_millis(100);
|
||||
loop {
|
||||
match tokio::net::TcpListener::bind(address).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(error)
|
||||
if error.kind() == std::io::ErrorKind::AddrInUse
|
||||
&& tokio::time::Instant::now() < deadline =>
|
||||
{
|
||||
tracing::warn!(target: "gateway_startup", address, "监听地址仍被占用,退避后重试绑定");
|
||||
tokio::time::sleep(delay).await;
|
||||
delay = (delay * 2).min(BIND_RETRY_MAX_DELAY);
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn serve_router(
|
||||
router: Router,
|
||||
host: &str,
|
||||
@@ -110,7 +142,7 @@ pub(crate) async fn serve_router(
|
||||
connections: Arc<ConnectionTracker>,
|
||||
drain_timeout: Duration,
|
||||
) -> Result<(), ServeError> {
|
||||
let listener = tokio::net::TcpListener::bind(format!("{host}:{port}"))
|
||||
let listener = bind_listener(&format!("{host}:{port}"))
|
||||
.await
|
||||
.map_err(ServeError::Bind)?;
|
||||
tracing::info!(target: "gateway_startup", host, port, "网关开始监听");
|
||||
@@ -121,16 +153,44 @@ pub(crate) async fn serve_router(
|
||||
wait_for_shutdown(&shutdown_cancellation).await;
|
||||
shutdown_connections.begin_draining();
|
||||
};
|
||||
axum::serve(
|
||||
// axum 的优雅关闭会无限等待存量连接结束,而 SSE 等长响应体不观察
|
||||
// 取消信号:慢客户端可以把关闭拖住直到被 SIGKILL。整个 drain(axum
|
||||
// 优雅关闭 + 游离 guard 等待)共享同一个从取消信号起算的绝对截止点,
|
||||
// 逾期即放弃等待,存量连接随进程退出终止。
|
||||
let drain_deadline_at = Arc::new(OnceLock::new());
|
||||
let drain_deadline = {
|
||||
let cancellation = cancellation.clone();
|
||||
let deadline_at = Arc::clone(&drain_deadline_at);
|
||||
async move {
|
||||
cancellation.cancelled().await;
|
||||
let deadline = tokio::time::Instant::now() + drain_timeout;
|
||||
let _ = deadline_at.set(deadline);
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
}
|
||||
};
|
||||
let serve = axum::serve(
|
||||
listener,
|
||||
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(shutdown)
|
||||
.await
|
||||
.map_err(ServeError::Serve)?;
|
||||
|
||||
connections.begin_draining();
|
||||
connections.wait_until_idle(drain_timeout).await;
|
||||
.with_graceful_shutdown(shutdown);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = serve => {
|
||||
result.map_err(ServeError::Serve)?;
|
||||
connections.begin_draining();
|
||||
let remaining = drain_deadline_at.get().map_or(drain_timeout, |deadline| {
|
||||
deadline.saturating_duration_since(tokio::time::Instant::now())
|
||||
});
|
||||
connections.wait_until_idle(remaining).await;
|
||||
}
|
||||
() = drain_deadline => {
|
||||
tracing::warn!(
|
||||
target: "gateway_shutdown",
|
||||
timeout_secs = drain_timeout.as_secs(),
|
||||
"优雅 drain 超时,放弃等待存量连接"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ pub(crate) async fn download_file(
|
||||
.map_err(|error| super::internal(format!("failed to create download file: {error}")))?;
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut downloaded = 0_u64;
|
||||
let mut last_percent = 0_u8;
|
||||
let mut next_progress = 10_u8;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|error| upstream(format!("download stream failed: {error}")))?;
|
||||
downloaded = downloaded.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
|
||||
@@ -59,9 +59,17 @@ pub(crate) async fn download_file(
|
||||
file.write_all(&chunk)
|
||||
.map_err(|error| super::internal(format!("failed to write download: {error}")))?;
|
||||
let percent = download_percent(downloaded, expected_size);
|
||||
if percent > last_percent {
|
||||
last_percent = percent;
|
||||
events.emit_progress(operation_id, "downloading release archive", percent);
|
||||
if percent >= next_progress || downloaded == expected_size {
|
||||
events.emit_progress(
|
||||
operation_id,
|
||||
&format!(
|
||||
"已下载 {} / {} ({percent}%)",
|
||||
format_bytes(downloaded),
|
||||
format_bytes(expected_size)
|
||||
),
|
||||
percent,
|
||||
);
|
||||
next_progress = percent.saturating_add(10);
|
||||
}
|
||||
}
|
||||
file.sync_all()
|
||||
@@ -155,3 +163,15 @@ fn download_percent(downloaded: u64, total: u64) -> u8 {
|
||||
)
|
||||
.unwrap_or(100)
|
||||
}
|
||||
|
||||
pub(crate) fn format_bytes(bytes: u64) -> String {
|
||||
const KIB: u64 = 1024;
|
||||
const MIB: u64 = KIB * 1024;
|
||||
if bytes >= MIB {
|
||||
return format!("{:.1} MiB", bytes as f64 / MIB as f64);
|
||||
}
|
||||
if bytes >= KIB {
|
||||
return format!("{:.1} KiB", bytes as f64 / KIB as f64);
|
||||
}
|
||||
format!("{bytes} B")
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ use self::download::{MAX_CHECKSUM_SIZE, MAX_DOWNLOAD_SIZE, download_file, verify
|
||||
use self::process::{environment_value, spawn_replacement};
|
||||
use self::release::{
|
||||
ReleaseCache, confirmed_target, detail_from_release, fetch_latest, select_archive,
|
||||
versions_cross_major,
|
||||
};
|
||||
use self::state::{
|
||||
OperationFileLock, UpdateTempDir, finish, operation_id, read_status, set_running,
|
||||
@@ -226,14 +227,32 @@ impl ProcessSystemOperations {
|
||||
return Err(conflict(reason));
|
||||
}
|
||||
let target = confirmed_target(target_version)?;
|
||||
if versions_cross_major(&self.config.version, &target).unwrap_or(false) {
|
||||
let reason = "跨大版本升级不提供自动更新,请按发布说明手动迁移";
|
||||
self.events.error_terminal(None, Some("preflight"), reason);
|
||||
return Err(conflict(reason));
|
||||
}
|
||||
let repository = self
|
||||
.config
|
||||
.update_repository
|
||||
.as_deref()
|
||||
.ok_or_else(|| conflict("update repository is not configured"))?;
|
||||
self.events
|
||||
.info(None, Some("release"), "fetching latest release");
|
||||
let release = fetch_latest(&self.config.github_api_base, repository).await?;
|
||||
.info(None, Some("release"), "正在获取最新 Release 信息");
|
||||
let release = match fetch_latest(
|
||||
&self.config.github_api_base,
|
||||
repository,
|
||||
&self.config.update_channel,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(release) => release,
|
||||
Err(error) => {
|
||||
self.events
|
||||
.error_terminal(None, Some("release"), error.to_string());
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let detail = detail_from_release(&self.config, &release);
|
||||
if detail.latest_version != target {
|
||||
self.events.warning_terminal(
|
||||
@@ -258,12 +277,17 @@ impl ProcessSystemOperations {
|
||||
Some(&target),
|
||||
&self.config.version,
|
||||
)?;
|
||||
self.events.info(
|
||||
Some(&operation_id),
|
||||
Some("prepare"),
|
||||
format!("准备更新到 v{target}"),
|
||||
);
|
||||
let result = self.install_release(&release, &target, &operation_id).await;
|
||||
match &result {
|
||||
Ok(()) => self.events.success_terminal(
|
||||
Some(&operation_id),
|
||||
Some("done"),
|
||||
"release files replaced",
|
||||
"更新文件已替换,等待服务重启生效",
|
||||
),
|
||||
Err(error) => {
|
||||
self.events
|
||||
@@ -294,7 +318,23 @@ impl ProcessSystemOperations {
|
||||
version: &str,
|
||||
operation_id: &str,
|
||||
) -> Result<(), OperationError> {
|
||||
self.events.info(
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
"正在选择匹配当前平台的更新包",
|
||||
);
|
||||
let archive = select_archive(release, version)?;
|
||||
self.events.info(
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
format!(
|
||||
"已选择更新包 {} ({})",
|
||||
archive.name,
|
||||
download::format_bytes(archive.size)
|
||||
),
|
||||
);
|
||||
self.events
|
||||
.info(Some(operation_id), Some("verify"), "正在校验更新资源");
|
||||
if archive.size == 0 || archive.size > MAX_DOWNLOAD_SIZE {
|
||||
return Err(invalid("release archive size is invalid"));
|
||||
}
|
||||
@@ -306,12 +346,16 @@ impl ProcessSystemOperations {
|
||||
if checksum.size == 0 || checksum.size > MAX_CHECKSUM_SIZE {
|
||||
return Err(invalid("release checksum size is invalid"));
|
||||
}
|
||||
self.events
|
||||
.info(Some(operation_id), Some("prepare"), "正在创建临时更新目录");
|
||||
fs::create_dir_all(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to prepare update temp dir: {error}")))?;
|
||||
let temp_root = fs::canonicalize(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to resolve update temp dir: {error}")))?;
|
||||
let temp = UpdateTempDir::create(&temp_root)?;
|
||||
let archive_path = temp.path().join(&archive.name);
|
||||
self.events
|
||||
.info(Some(operation_id), Some("download"), "开始下载更新包");
|
||||
download_file(
|
||||
&archive.browser_download_url,
|
||||
&archive_path,
|
||||
@@ -321,6 +365,10 @@ impl ProcessSystemOperations {
|
||||
&self.events,
|
||||
)
|
||||
.await?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("download"), "更新包下载完成");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("checksum"), "正在校验 checksum");
|
||||
verify_checksum(
|
||||
&archive_path,
|
||||
&archive.name,
|
||||
@@ -329,12 +377,23 @@ impl ProcessSystemOperations {
|
||||
&self.config.github_api_base,
|
||||
)
|
||||
.await?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("checksum"), "checksum 校验通过");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("extract"), "正在解压更新包");
|
||||
let extracted = extract_release(&archive_path, temp.path())?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("extract"), "更新包解压完成");
|
||||
self.events
|
||||
.info(Some(operation_id), Some("replace"), "正在替换应用文件");
|
||||
replace_release_files(
|
||||
&self.config.executable_path()?,
|
||||
&self.config.web_dist_dir,
|
||||
extracted,
|
||||
)
|
||||
)?;
|
||||
self.events
|
||||
.success(Some(operation_id), Some("replace"), "应用文件替换完成");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,6 +501,12 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
}
|
||||
|
||||
async fn restart(&self) -> Result<SystemOperationAccepted, OperationError> {
|
||||
// 与 update/rollback 互斥:更新替换文件期间触发自重启会让新进程
|
||||
// 载入半成品产物。
|
||||
let _operation = self
|
||||
.operation_lock
|
||||
.try_lock()
|
||||
.map_err(|_| conflict("system operation is already running"))?;
|
||||
if !self.config.self_restart_enabled {
|
||||
return Err(conflict("self restart is disabled"));
|
||||
}
|
||||
@@ -515,6 +580,17 @@ impl UpdateEvents {
|
||||
);
|
||||
}
|
||||
|
||||
fn success(&self, operation_id: Option<&str>, step: Option<&str>, message: impl Into<String>) {
|
||||
self.emit(
|
||||
SystemUpdateEventLevel::Success,
|
||||
operation_id,
|
||||
step,
|
||||
message,
|
||||
None,
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
fn warning_terminal(
|
||||
&self,
|
||||
operation_id: Option<&str>,
|
||||
|
||||
@@ -20,6 +20,8 @@ pub(crate) struct GitHubRelease {
|
||||
pub(crate) html_url: Option<String>,
|
||||
pub(crate) prerelease: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) draft: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) assets: Vec<GitHubAsset>,
|
||||
}
|
||||
|
||||
@@ -57,7 +59,7 @@ impl ReleaseCache {
|
||||
if !refresh && let Some(detail) = self.cached(&key).await {
|
||||
return Ok(detail);
|
||||
}
|
||||
match fetch_latest(&config.github_api_base, repository).await {
|
||||
match fetch_latest(&config.github_api_base, repository, &config.update_channel).await {
|
||||
Ok(release) => {
|
||||
let detail = detail_from_release(config, &release);
|
||||
*self.entry.lock().await = Some(CachedRelease {
|
||||
@@ -85,13 +87,27 @@ impl ReleaseCache {
|
||||
pub(crate) async fn fetch_latest(
|
||||
api_base: &str,
|
||||
repository: &str,
|
||||
update_channel: &str,
|
||||
) -> Result<GitHubRelease, OperationError> {
|
||||
validate_api_base(api_base).map_err(conflict)?;
|
||||
validate_repository(repository)?;
|
||||
let url = format!(
|
||||
"{}/{repository}/releases/latest",
|
||||
api_base.trim_end_matches('/')
|
||||
);
|
||||
let base = api_base.trim_end_matches('/');
|
||||
// GitHub 的 /releases/latest 永远只返回正式版;preview 渠道需要覆盖
|
||||
// prerelease,因此改查列表端点并取最新一条非草稿 Release。
|
||||
if update_channel == "preview" {
|
||||
let releases: Vec<GitHubRelease> =
|
||||
fetch_release_json(&format!("{base}/{repository}/releases?per_page=10")).await?;
|
||||
return releases
|
||||
.into_iter()
|
||||
.find(|release| !release.draft)
|
||||
.ok_or_else(|| upstream("no published release available"));
|
||||
}
|
||||
fetch_release_json(&format!("{base}/{repository}/releases/latest")).await
|
||||
}
|
||||
|
||||
async fn fetch_release_json<T: serde::de::DeserializeOwned>(
|
||||
url: &str,
|
||||
) -> Result<T, OperationError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
@@ -111,7 +127,7 @@ pub(crate) async fn fetch_latest(
|
||||
)));
|
||||
}
|
||||
response
|
||||
.json::<GitHubRelease>()
|
||||
.json::<T>()
|
||||
.await
|
||||
.map_err(|error| upstream(format!("invalid GitHub release response: {error}")))
|
||||
}
|
||||
@@ -121,12 +137,13 @@ pub(crate) fn detail_from_release(
|
||||
release: &GitHubRelease,
|
||||
) -> SystemUpdateDetail {
|
||||
let latest = normalize_version(&release.tag_name);
|
||||
let available = (config.update_channel != "stable" || !release.prerelease)
|
||||
let newer = (config.update_channel != "stable" || !release.prerelease)
|
||||
&& version_is_newer(&config.version, &latest).unwrap_or(false);
|
||||
let cross_major = versions_cross_major(&config.version, &latest).unwrap_or(false);
|
||||
SystemUpdateDetail {
|
||||
current_version: config.version.clone(),
|
||||
latest_version: latest,
|
||||
has_update: available,
|
||||
latest_version: latest.clone(),
|
||||
has_update: newer && !cross_major,
|
||||
deployment_mode: config.deployment_mode.clone(),
|
||||
build_type: config.build_type.clone(),
|
||||
release_url: release.html_url.clone(),
|
||||
@@ -134,7 +151,9 @@ pub(crate) fn detail_from_release(
|
||||
cached: false,
|
||||
update_supported: config.update_support_error().is_none(),
|
||||
unsupported_reason: config.update_support_error(),
|
||||
warning: None,
|
||||
warning: (newer && cross_major).then(|| {
|
||||
format!("检测到新的大版本 v{latest}:跨大版本升级不提供自动更新,请按发布说明手动迁移")
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,6 +284,13 @@ fn version_is_newer(current: &str, latest: &str) -> Option<bool> {
|
||||
Some(latest > current)
|
||||
}
|
||||
|
||||
/// 自动更新只在同 major 内提供;跨大版本的配置与数据迁移契约不保证兼容。
|
||||
pub(crate) fn versions_cross_major(current: &str, latest: &str) -> Option<bool> {
|
||||
let current = semver::Version::parse(&normalize_version(current)).ok()?;
|
||||
let latest = semver::Version::parse(&normalize_version(latest)).ok()?;
|
||||
Some(current.major != latest.major)
|
||||
}
|
||||
|
||||
fn repository_character(character: char) -> bool {
|
||||
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')
|
||||
}
|
||||
@@ -272,7 +298,6 @@ fn repository_character(character: char) -> bool {
|
||||
fn platform_os_aliases() -> &'static [&'static str] {
|
||||
match env::consts::OS {
|
||||
"macos" => &["macos", "darwin"],
|
||||
"windows" => &["windows", "win32"],
|
||||
"linux" => &["linux"],
|
||||
_ => &[env::consts::OS],
|
||||
}
|
||||
|
||||
@@ -108,22 +108,19 @@ impl WorkerSupervisor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 先协作取消;超时后 abort 不合作任务,lease 依赖 TTL 兜底。
|
||||
/// 先协作取消;所有任务共享同一绝对截止点,逾期任务被 abort,
|
||||
/// lease 依赖 TTL 兜底。每个句柄恰好 join 一次:已在截止点前退出的
|
||||
/// 不会被二次 poll,逾期的在 abort 后等待其真正终止。
|
||||
pub async fn shutdown(&self, timeout: Duration) {
|
||||
self.cancellation.cancel();
|
||||
let mut handles = std::mem::take(&mut *lock_unpoisoned(&self.handles));
|
||||
let joined = tokio::time::timeout(timeout, async {
|
||||
for (_, handle) in &mut handles {
|
||||
let _ = handle.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
if !joined {
|
||||
for (_, handle) in &handles {
|
||||
let handles = std::mem::take(&mut *lock_unpoisoned(&self.handles));
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
for (_, mut handle) in handles {
|
||||
if tokio::time::timeout_at(deadline, &mut handle)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
handle.abort();
|
||||
}
|
||||
for (_, handle) in handles {
|
||||
let _ = handle.await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::engine::CancellationToken;
|
||||
use gateway_core::lifecycle::ConnectionLifecycle as _;
|
||||
use gateway_host::HostBundle;
|
||||
use gateway_host::serve::{ConnectionTracker, bind_listener};
|
||||
|
||||
#[test]
|
||||
fn host_bundle_serve_is_a_consuming_process_entrypoint() {
|
||||
@@ -6,3 +12,83 @@ fn host_bundle_serve_is_a_consuming_process_entrypoint() {
|
||||
|
||||
assert_eq!(std::mem::size_of_val(&_serve), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_until_idle_should_return_immediately_without_active_connections() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
tracker.wait_until_idle(Duration::from_secs(30)).await;
|
||||
|
||||
assert_eq!(started.elapsed(), Duration::ZERO);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_until_idle_should_give_up_at_timeout_while_connections_remain() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let _guard = tracker.try_register().expect("register connection");
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
tracker.wait_until_idle(Duration::from_secs(30)).await;
|
||||
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(30));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_until_idle_should_wake_when_last_guard_drops_after_first_poll() {
|
||||
let tracker = ConnectionTracker::new(CancellationToken::new());
|
||||
let guard = tracker.try_register().expect("register connection");
|
||||
let mut wait = Box::pin(tracker.wait_until_idle(Duration::from_secs(30)));
|
||||
assert!(futures::poll!(wait.as_mut()).is_pending());
|
||||
|
||||
drop(guard);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), wait)
|
||||
.await
|
||||
.expect("woken by last guard drop instead of waiting out the timeout");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn wait_until_idle_should_observe_guard_drop_racing_the_idle_check() {
|
||||
// 回归约束:notified 必须在读取活跃计数前完成注册(enable),否则最后
|
||||
// 一个 guard 在计数检查与首次 poll 之间 drop 时唤醒丢失,只能等满整个
|
||||
// 超时。多线程反复交错,任何一次丢唤醒都会撞上 2s 超时并触发断言。
|
||||
for _ in 0..256 {
|
||||
let tracker = Arc::new(ConnectionTracker::new(CancellationToken::new()));
|
||||
let guard = tracker.try_register().expect("register connection");
|
||||
let waiter = tokio::spawn({
|
||||
let tracker = Arc::clone(&tracker);
|
||||
async move { tracker.wait_until_idle(Duration::from_secs(2)).await }
|
||||
});
|
||||
let dropper = std::thread::spawn(move || drop(guard));
|
||||
let started = std::time::Instant::now();
|
||||
waiter.await.expect("waiter completes");
|
||||
dropper.join().expect("dropper completes");
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(1_900),
|
||||
"idle wakeup was lost and the waiter slept until the drain timeout"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bind_listener_should_retry_until_previous_listener_releases_port() {
|
||||
let holder = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("holder listener");
|
||||
let address = holder.local_addr().expect("holder address").to_string();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
drop(holder);
|
||||
});
|
||||
|
||||
let listener = tokio::time::timeout(Duration::from_secs(8), bind_listener(&address))
|
||||
.await
|
||||
.expect("bound within the retry window")
|
||||
.expect("bound after the previous listener released the port");
|
||||
|
||||
assert_eq!(
|
||||
listener.local_addr().expect("bound address").to_string(),
|
||||
address
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use filetime::FileTime;
|
||||
@@ -18,7 +19,8 @@ use tar::{Builder, EntryType, Header};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
const TARGET_VERSION: &str = "9.9.9";
|
||||
const TARGET_VERSION: &str = "1.9.9";
|
||||
const CROSS_MAJOR_VERSION: &str = "2.0.0";
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_not_shutdown_when_replacement_spawn_fails() {
|
||||
@@ -78,6 +80,49 @@ async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
.expect("replacement executed before fixture cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_conflict_while_another_system_operation_is_running() {
|
||||
let fixture = Fixture::new();
|
||||
// 裸 TCP 监听不回包:update 持有操作锁后停在 release 拉取阶段,accept
|
||||
// 信号保证断言时锁一定已被占用。
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("release listener");
|
||||
let api_base = format!("http://{}/repos", listener.local_addr().expect("addr"));
|
||||
let (connected_tx, connected_rx) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
let connection = listener.accept().await;
|
||||
let _ = connected_tx.send(());
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
drop(connection);
|
||||
});
|
||||
let mut config = fixture.config(&api_base);
|
||||
config.self_restart_enabled = true;
|
||||
config.deployment_mode = "docker".to_owned();
|
||||
let shutdown = CancellationToken::new();
|
||||
let service = Arc::new(ProcessSystemOperations::new(shutdown.clone(), config));
|
||||
let update = tokio::spawn({
|
||||
let service = Arc::clone(&service);
|
||||
async move {
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), connected_rx)
|
||||
.await
|
||||
.expect("update reaches release fetch while holding the operation lock")
|
||||
.expect("connected signal");
|
||||
|
||||
let error = service.restart().await.expect_err("restart during update");
|
||||
assert_eq!(error.kind(), SystemOperationErrorKind::Conflict);
|
||||
assert!(!shutdown.is_cancelled());
|
||||
|
||||
update.abort();
|
||||
let _ = update.await;
|
||||
service.restart().await.expect("restart after lock release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_should_restore_binary_web_and_version_state() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -134,6 +179,40 @@ async fn update_detail_should_fallback_to_cache_when_refresh_fetch_fails() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_offer_latest_non_draft_prerelease_on_preview_channel() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/repos/owner/repository/releases"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
|
||||
{
|
||||
"tag_name": "v1.2.0-beta.2",
|
||||
"name": "Draft 1.2.0-beta.2",
|
||||
"prerelease": true,
|
||||
"draft": true,
|
||||
"assets": [],
|
||||
},
|
||||
{
|
||||
"tag_name": "v1.1.0-beta.1",
|
||||
"name": "Release 1.1.0-beta.1",
|
||||
"body": "notes",
|
||||
"html_url": "https://github.com/owner/repository/releases",
|
||||
"prerelease": true,
|
||||
"assets": [],
|
||||
},
|
||||
])))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let mut config = fixture.config(&format!("{}/repos", server.uri()));
|
||||
config.update_channel = "preview".to_owned();
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
|
||||
|
||||
let detail = service.update_detail(true).await.expect("detail");
|
||||
assert_eq!(detail.latest_version, "1.1.0-beta.1");
|
||||
assert!(detail.has_update);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_reject_untrusted_github_api_base() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -174,6 +253,43 @@ async fn update_detail_should_use_cached_release_when_not_refreshed() {
|
||||
assert!(service.update_detail(false).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_withhold_cross_major_release() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release_once(&server, CROSS_MAJOR_VERSION)
|
||||
.await;
|
||||
|
||||
let detail = fixture
|
||||
.service(&server)
|
||||
.update_detail(true)
|
||||
.await
|
||||
.expect("detail");
|
||||
assert_eq!(detail.latest_version, CROSS_MAJOR_VERSION);
|
||||
assert!(!detail.has_update);
|
||||
assert!(detail.warning.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_reject_cross_major_target_before_fetching_release() {
|
||||
let fixture = Fixture::new();
|
||||
let service = ProcessSystemOperations::new(
|
||||
CancellationToken::new(),
|
||||
fixture.config("http://127.0.0.1:1/repos"),
|
||||
);
|
||||
|
||||
let error = service
|
||||
.perform_update(Some(CROSS_MAJOR_VERSION.to_owned()))
|
||||
.await
|
||||
.expect_err("cross-major target must be rejected");
|
||||
assert_eq!(error.kind(), SystemOperationErrorKind::Conflict);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("binary"),
|
||||
b"old-binary"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_close_after_terminal_update_log() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -204,6 +320,68 @@ async fn update_events_should_open_authenticated_sse_stream() {
|
||||
assert!(stream.next().await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_preserve_complete_release_stage_sequence() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let events = service.update_events();
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
let steps = events
|
||||
.map(|event| event.step.expect("release update step"))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
steps,
|
||||
[
|
||||
"release", "prepare", "asset", "asset", "verify", "prepare", "download", "download",
|
||||
"download", "checksum", "checksum", "extract", "extract", "replace", "replace", "done",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_events_should_report_download_bytes_and_percent() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let events = service.update_events();
|
||||
service
|
||||
.perform_update(Some(TARGET_VERSION.to_owned()))
|
||||
.await
|
||||
.expect("update");
|
||||
let progress = events
|
||||
.filter_map(
|
||||
|event| async move { event.progress_percent.map(|value| (value, event.message)) },
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(progress.iter().any(|(percent, message)| {
|
||||
*percent == 100 && message.starts_with("已下载 ") && message.ends_with(" (100%)")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_fail_when_release_checksum_is_missing() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -335,7 +513,7 @@ async fn update_should_reject_when_confirmed_target_differs_from_remote_latest()
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(&server, "9.9.8", ArchiveKind::Safe, ChecksumKind::Valid)
|
||||
.mount_release(&server, "1.9.8", ArchiveKind::Safe, ChecksumKind::Valid)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -626,6 +626,55 @@ async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() {
|
||||
// 挂死任务放在 join 顺序最末的 kind(Retention),保证超时触发时
|
||||
// 排在它前面的合作任务句柄都已被 join 过一次。
|
||||
let hung = HungTask {
|
||||
entered: Arc::new(AtomicBool::new(false)),
|
||||
notification: Arc::new(Notify::new()),
|
||||
};
|
||||
let cooperative = CancellationTask {
|
||||
entered: Arc::new(AtomicUsize::new(0)),
|
||||
observed: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
let mut plan = ACTIVE_KINDS
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
if kind == WorkerKind::Retention {
|
||||
registration(kind, kind.as_str(), hung.clone(), test_schedule(), true)
|
||||
} else {
|
||||
registration(
|
||||
kind,
|
||||
kind.as_str(),
|
||||
cooperative.clone(),
|
||||
test_schedule(),
|
||||
true,
|
||||
)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
plan.push(WorkerContribution::Disabled {
|
||||
kind: WorkerKind::OpsFlush,
|
||||
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
||||
});
|
||||
let leases = Arc::new(FakeLeasePort::default());
|
||||
let supervisor = supervisor();
|
||||
supervisor
|
||||
.start(plan, leases.clone())
|
||||
.expect("complete plan");
|
||||
hung.notification.notified().await;
|
||||
wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await;
|
||||
|
||||
supervisor.shutdown(Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(
|
||||
cooperative.observed.load(Ordering::SeqCst),
|
||||
ACTIVE_KINDS.len() - 1
|
||||
);
|
||||
assert!(lock_unpoisoned(&leases.active_guards).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn running_worker_becomes_unhealthy_when_last_success_is_stale() {
|
||||
let task = SucceedsThenHangsTask {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const MULTI_AGENT_MODE_OPEN_TAG: &str = "<multi_agent_mode>";
|
||||
const MULTI_AGENT_MODE_CLOSE_TAG: &str = "</multi_agent_mode>";
|
||||
const PROACTIVE_MULTI_AGENT_MODE_PREFIX: &str = "Proactive multi-agent delegation is active.";
|
||||
|
||||
/// 从 OpenAI Responses 请求中提取的稳定 Codex 请求语义。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CodexResponsesRequestSemantics {
|
||||
/// 客户端原始选择的推理强度。
|
||||
pub reasoning_effort: Option<String>,
|
||||
/// Codex turn metadata 中的请求类型。
|
||||
pub request_kind: Option<String>,
|
||||
/// Codex turn metadata 中的子代理类型。
|
||||
pub subagent_kind: Option<String>,
|
||||
/// 网关侧派生出的推理预设。
|
||||
pub reasoning_preset: Option<&'static str>,
|
||||
/// 请求是否为 Codex 压缩请求。
|
||||
pub compact: bool,
|
||||
}
|
||||
|
||||
/// 从 Responses body 与非 wire 协议上下文中提取 Codex 语义。
|
||||
#[must_use]
|
||||
pub fn codex_responses_request_semantics(
|
||||
body: &Map<String, Value>,
|
||||
context: &Map<String, Value>,
|
||||
) -> CodexResponsesRequestSemantics {
|
||||
codex_responses_request_semantics_with_turn_metadata(
|
||||
body,
|
||||
non_empty_string(context.get("turn_metadata")),
|
||||
)
|
||||
}
|
||||
|
||||
/// 调用方已解析权威 turn metadata 时,从 Responses body 中提取 Codex 语义。
|
||||
#[must_use]
|
||||
pub fn codex_responses_request_semantics_with_turn_metadata(
|
||||
body: &Map<String, Value>,
|
||||
turn_metadata: Option<&str>,
|
||||
) -> CodexResponsesRequestSemantics {
|
||||
let reasoning_effort = body
|
||||
.get("reasoning")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|reasoning| non_empty_string(reasoning.get("effort")))
|
||||
.map(ToOwned::to_owned);
|
||||
let parsed_turn_metadata = turn_metadata
|
||||
.or_else(|| request_turn_metadata(body))
|
||||
.and_then(|value| serde_json::from_str::<Value>(value).ok());
|
||||
let request_kind = parsed_turn_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| non_empty_string(metadata.get("request_kind")))
|
||||
.map(ToOwned::to_owned);
|
||||
let subagent_kind = parsed_turn_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| non_empty_string(metadata.get("subagent_kind")))
|
||||
.map(ToOwned::to_owned);
|
||||
let proactive_multi_agent = latest_multi_agent_mode(input_items(body).iter())
|
||||
.is_some_and(|mode| mode.starts_with(PROACTIVE_MULTI_AGENT_MODE_PREFIX));
|
||||
let compact_trigger = input_items(body)
|
||||
.iter()
|
||||
.any(|item| non_empty_string(item.get("type")) == Some("compaction_trigger"));
|
||||
let reasoning_preset = (subagent_kind.is_none()
|
||||
&& reasoning_effort.as_deref() == Some("max")
|
||||
&& proactive_multi_agent)
|
||||
.then_some("ultra");
|
||||
let compact = request_kind.as_deref() == Some("compaction") || compact_trigger;
|
||||
|
||||
CodexResponsesRequestSemantics {
|
||||
reasoning_effort,
|
||||
request_kind,
|
||||
subagent_kind,
|
||||
reasoning_preset,
|
||||
compact,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_turn_metadata(body: &Map<String, Value>) -> Option<&str> {
|
||||
non_empty_string(body.get("turnMetadata"))
|
||||
.or_else(|| non_empty_string(body.get("turn_metadata")))
|
||||
.or_else(|| non_empty_string(body.get("x-codex-turn-metadata")))
|
||||
.or_else(|| client_metadata_string(body, "x-codex-turn-metadata"))
|
||||
.or_else(|| client_metadata_string(body, "turnMetadata"))
|
||||
.or_else(|| client_metadata_string(body, "turn_metadata"))
|
||||
}
|
||||
|
||||
fn client_metadata_string<'a>(body: &'a Map<String, Value>, key: &str) -> Option<&'a str> {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| non_empty_string(metadata.get(key)))
|
||||
}
|
||||
|
||||
fn input_items(body: &Map<String, Value>) -> &[Value] {
|
||||
body.get("input")
|
||||
.and_then(Value::as_array)
|
||||
.map_or(&[], Vec::as_slice)
|
||||
}
|
||||
|
||||
fn latest_multi_agent_mode<'a>(
|
||||
input: impl DoubleEndedIterator<Item = &'a Value>,
|
||||
) -> Option<&'a str> {
|
||||
input.rev().find_map(|item| {
|
||||
if non_empty_string(item.get("role")) != Some("developer") {
|
||||
return None;
|
||||
}
|
||||
item.get("content")?
|
||||
.as_array()?
|
||||
.iter()
|
||||
.rev()
|
||||
.filter_map(|content| non_empty_string(content.get("text")))
|
||||
.find_map(multi_agent_mode_from_text)
|
||||
})
|
||||
}
|
||||
|
||||
fn multi_agent_mode_from_text(text: &str) -> Option<&str> {
|
||||
let close = text.rfind(MULTI_AGENT_MODE_CLOSE_TAG)?;
|
||||
let before_close = &text[..close];
|
||||
let open = before_close.rfind(MULTI_AGENT_MODE_OPEN_TAG)?;
|
||||
let body = &before_close[open + MULTI_AGENT_MODE_OPEN_TAG.len()..];
|
||||
Some(body.trim())
|
||||
}
|
||||
|
||||
fn non_empty_string(value: Option<&Value>) -> Option<&str> {
|
||||
value
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -9,7 +9,14 @@ pub const WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY: &str =
|
||||
/// Codex memory consolidation 请求标记。
|
||||
pub const X_OPENAI_MEMGEN_REQUEST_HEADER: &str = "x-openai-memgen-request";
|
||||
|
||||
/// OpenAI Responses 请求中携带的 Codex 专属语义。
|
||||
pub mod codex;
|
||||
/// OpenAI/Codex 事件语义、用量与限流字段编解码。
|
||||
pub mod events;
|
||||
/// Server-Sent Events 帧的解析与编码。
|
||||
pub mod sse;
|
||||
|
||||
pub use codex::{
|
||||
CodexResponsesRequestSemantics, codex_responses_request_semantics,
|
||||
codex_responses_request_semantics_with_turn_metadata,
|
||||
};
|
||||
|
||||
@@ -72,26 +72,50 @@ pub const DONE_SSE_FRAME: &str = "data: [DONE]\n\n";
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SseEventDecoder {
|
||||
pending: Vec<u8>,
|
||||
/// `pending` 中已确认不含帧分隔符的前缀长度。新数据到达时从该边界
|
||||
/// 回退分隔符最大跨界宽度继续扫描,大帧跨多个 chunk 不会被反复重扫。
|
||||
scanned: usize,
|
||||
}
|
||||
|
||||
impl SseEventDecoder {
|
||||
/// 返回从 `consumed` 起的下一个完整帧结束位置(含分隔符),并推进扫描边界。
|
||||
fn next_frame_end(&mut self, consumed: usize) -> Option<usize> {
|
||||
// `\r\n\r\n` 分隔符可能跨 chunk 边界,扫描起点回退 3 字节。
|
||||
let from = self.scanned.saturating_sub(3).max(consumed);
|
||||
match sse_frame_separator_bytes(&self.pending[from..]) {
|
||||
Some((position, separator_len)) => {
|
||||
let end = from + position + separator_len;
|
||||
self.scanned = end;
|
||||
Some(end)
|
||||
}
|
||||
None => {
|
||||
self.scanned = self.pending.len();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_consumed(&mut self, consumed: usize) {
|
||||
if consumed != 0 {
|
||||
self.pending.drain(..consumed);
|
||||
self.scanned = self.scanned.saturating_sub(consumed);
|
||||
}
|
||||
}
|
||||
|
||||
/// 追加一个字节块并返回其中已经完整的事件。
|
||||
pub fn push(&mut self, chunk: &[u8]) -> Result<Vec<SseEvent>, SseError> {
|
||||
self.pending.extend_from_slice(chunk);
|
||||
let mut events = Vec::new();
|
||||
let mut consumed = 0usize;
|
||||
|
||||
while let Some(frame_len) = sse_frame_end(&self.pending[consumed..]) {
|
||||
let end = consumed.saturating_add(frame_len);
|
||||
while let Some(end) = self.next_frame_end(consumed) {
|
||||
let frame = std::str::from_utf8(&self.pending[consumed..end])
|
||||
.map_err(|error| SseError::ParseError(error.to_string()))?;
|
||||
events.extend(parse_sse_events(frame)?);
|
||||
consumed = end;
|
||||
}
|
||||
|
||||
if consumed != 0 {
|
||||
self.pending.drain(..consumed);
|
||||
}
|
||||
self.commit_consumed(consumed);
|
||||
if self.pending.len() > MAX_SSE_EVENT_BUFFER_BYTES {
|
||||
return Err(SseError::BufferExceeded {
|
||||
max_bytes: MAX_SSE_EVENT_BUFFER_BYTES,
|
||||
@@ -110,8 +134,7 @@ impl SseEventDecoder {
|
||||
let mut frames = Vec::new();
|
||||
let mut consumed = 0usize;
|
||||
|
||||
while let Some(frame_len) = sse_frame_end(&self.pending[consumed..]) {
|
||||
let end = consumed.saturating_add(frame_len);
|
||||
while let Some(end) = self.next_frame_end(consumed) {
|
||||
let raw = self.pending[consumed..end].to_vec();
|
||||
let events = std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
@@ -121,9 +144,7 @@ impl SseEventDecoder {
|
||||
consumed = end;
|
||||
}
|
||||
|
||||
if consumed != 0 {
|
||||
self.pending.drain(..consumed);
|
||||
}
|
||||
self.commit_consumed(consumed);
|
||||
if self.pending.len() > MAX_SSE_EVENT_BUFFER_BYTES {
|
||||
return Err(SseError::BufferExceeded {
|
||||
max_bytes: MAX_SSE_EVENT_BUFFER_BYTES,
|
||||
@@ -134,6 +155,7 @@ impl SseEventDecoder {
|
||||
|
||||
/// 流结束时解析尚未带空行分隔符的最后一帧。
|
||||
pub fn finish(&mut self) -> Result<Vec<SseEvent>, SseError> {
|
||||
self.scanned = 0;
|
||||
if self.pending.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -147,6 +169,7 @@ impl SseEventDecoder {
|
||||
///
|
||||
/// 与 [`Self::push_frames`] 一致,旁路解析失败不会丢弃原始帧。
|
||||
pub fn finish_frames(&mut self) -> Result<Vec<SseFrame>, SseError> {
|
||||
self.scanned = 0;
|
||||
if self.pending.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -235,9 +258,6 @@ pub enum SseError {
|
||||
/// 解析错误。
|
||||
#[error("SSE parse error: {0}")]
|
||||
ParseError(String),
|
||||
/// 用量提取错误。
|
||||
#[error("Usage extraction error: {0}")]
|
||||
UsageExtraction(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
use gateway_protocol::openai::codex_responses_request_semantics;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
#[test]
|
||||
fn codex_semantics_should_use_protocol_context_before_body_metadata() {
|
||||
let body = json!({
|
||||
"model": "grok-4.5",
|
||||
"reasoning": {"effort": " xhigh "},
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": "{\"request_kind\":\"review\"}"
|
||||
},
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": "history"},
|
||||
{"type": "compaction_trigger"}
|
||||
]
|
||||
});
|
||||
let body = object(body);
|
||||
let context = Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
json!("{\"request_kind\":\"compaction\",\"subagent_kind\":\"compact\"}"),
|
||||
)]);
|
||||
|
||||
let semantics = codex_responses_request_semantics(&body, &context);
|
||||
|
||||
assert_eq!(semantics.reasoning_effort.as_deref(), Some("xhigh"));
|
||||
assert_eq!(semantics.request_kind.as_deref(), Some("compaction"));
|
||||
assert_eq!(semantics.subagent_kind.as_deref(), Some("compact"));
|
||||
assert_eq!(semantics.reasoning_preset, None);
|
||||
assert!(semantics.compact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_semantics_should_detect_ultra_preset_from_proactive_multi_agent() {
|
||||
let body = object(json!({
|
||||
"model": "gpt-test",
|
||||
"reasoning": {"effort": "max"},
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": "<multi_agent_mode>Proactive multi-agent delegation is active.</multi_agent_mode>"
|
||||
}]
|
||||
}
|
||||
]
|
||||
}));
|
||||
|
||||
let semantics = codex_responses_request_semantics(&body, &Map::new());
|
||||
|
||||
assert_eq!(semantics.reasoning_effort.as_deref(), Some("max"));
|
||||
assert_eq!(semantics.reasoning_preset, Some("ultra"));
|
||||
assert!(!semantics.compact);
|
||||
}
|
||||
|
||||
fn object(value: Value) -> Map<String, Value> {
|
||||
let Value::Object(object) = value else {
|
||||
panic!("test value must be an object");
|
||||
};
|
||||
object
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
mod codex;
|
||||
mod events;
|
||||
mod sse;
|
||||
|
||||
@@ -107,6 +107,29 @@ fn incremental_decoder_should_decode_across_arbitrary_chunk_boundaries() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incremental_decoder_should_find_separators_split_at_every_byte_boundary() {
|
||||
// CRLF 分隔符按单字节到达时必须仍被识别;帧内容跨任意多次 push 不丢失。
|
||||
let body = b"event: one\r\ndata: first\r\n\r\nevent: two\ndata: second\n\ndata: third\r\n\r\n";
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
let mut events = Vec::new();
|
||||
for byte in body {
|
||||
events.extend(decoder.push(&[*byte]).expect("single-byte chunk"));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.map(|event| (event.event.as_deref(), event.data.as_str()))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(Some("one"), "first"),
|
||||
(Some("two"), "second"),
|
||||
(None, "third"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_frame_decoder_should_preserve_exact_frame_across_chunk_boundaries() {
|
||||
let raw = b": keep-alive\r\nid: evt_1\r\nevent: response.future\r\nretry: 250\r\ndata: { \"type\": \"response.future\", \"opaque\": true }\r\n\r\n";
|
||||
|
||||
@@ -86,6 +86,37 @@ pub type StoreResult<T> = Result<T, StoreError>;
|
||||
pub struct StoreConfig {
|
||||
database: StoreConnectionConfig,
|
||||
redis: StoreConnectionConfig,
|
||||
#[serde(default)]
|
||||
pool: StorePoolConfig,
|
||||
}
|
||||
|
||||
/// PostgreSQL 连接池预算;acquire 超时决定池耗尽时快速失败而非排队积压。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct StorePoolConfig {
|
||||
pub max_connections: u32,
|
||||
pub acquire_timeout_seconds: u64,
|
||||
}
|
||||
|
||||
impl Default for StorePoolConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_connections: 10,
|
||||
acquire_timeout_seconds: 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StorePoolConfig {
|
||||
fn validate(&self) -> StoreResult<()> {
|
||||
if self.max_connections == 0 || self.acquire_timeout_seconds == 0 {
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: "pool limits must be positive".to_owned(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StoreConfig {
|
||||
@@ -104,6 +135,7 @@ impl StoreConfig {
|
||||
}
|
||||
self.database.validate("database")?;
|
||||
self.redis.validate("redis")?;
|
||||
self.pool.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -227,7 +259,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
const REDIS_NAMESPACE: &str = "codex-proxy-rs";
|
||||
|
||||
config.resolve_and_validate(std::path::Path::new("."))?;
|
||||
let pool = postgres::connect_and_migrate(&config.database_url()?).await?;
|
||||
let pool = postgres::connect_and_migrate(&config.database_url()?, config.pool).await?;
|
||||
let redis_client = ::redis::Client::open(config.redis_url()?)
|
||||
.map_err(|_| redis_unavailable("create Redis client"))?;
|
||||
let redis_connection = redis_client
|
||||
@@ -550,7 +582,6 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
};
|
||||
let snapshot = postgres::ControlPlaneRepository::replace_control_plane(
|
||||
&self.control_plane,
|
||||
store_revision(command.expected_config_revision)?,
|
||||
replacement,
|
||||
)
|
||||
.await
|
||||
@@ -560,32 +591,24 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: AdminRevision,
|
||||
key: AdminApiKey,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
self.replace_admin_api_key_value(
|
||||
expected_config_revision,
|
||||
Some(key.expose_for_auth().to_owned()),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
self.replace_admin_api_key_value(Some(key.expose_for_auth().to_owned()), context)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
expected_config_revision: AdminRevision,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
self.replace_admin_api_key_value(expected_config_revision, None, context)
|
||||
.await
|
||||
self.replace_admin_api_key_value(None, context).await
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminSettingsStoreAdapter {
|
||||
async fn replace_admin_api_key_value(
|
||||
&self,
|
||||
expected_config_revision: AdminRevision,
|
||||
admin_api_key: Option<String>,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
@@ -607,7 +630,6 @@ impl AdminSettingsStoreAdapter {
|
||||
};
|
||||
let snapshot = postgres::ControlPlaneRepository::replace_control_plane(
|
||||
&self.control_plane,
|
||||
store_revision(expected_config_revision)?,
|
||||
postgres::ControlPlaneReplacement {
|
||||
settings: update,
|
||||
audit: mutation_audit(
|
||||
@@ -823,44 +845,6 @@ impl AuthStore for AdminAuthStoreAdapter {
|
||||
.map_err(|error| admin_store_error("admin session", error))
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
redis::AdminAuthStateRepository::login_source_is_throttled(
|
||||
&self.state,
|
||||
source,
|
||||
failure_limit,
|
||||
window_seconds,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin login throttle", error))
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> AdminStoreResult<bool> {
|
||||
redis::AdminAuthStateRepository::record_login_failure(
|
||||
&self.state,
|
||||
source,
|
||||
failure_limit,
|
||||
window_seconds,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin login throttle", error))
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> AdminStoreResult<()> {
|
||||
redis::AdminAuthStateRepository::clear_login_failures(&self.state, source)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin login throttle", error))
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditModel) -> AdminStoreResult<()> {
|
||||
let config_revision = event
|
||||
.config_revision
|
||||
|
||||
@@ -36,7 +36,7 @@ use sqlx::{PgPool, Postgres, QueryBuilder, Transaction};
|
||||
|
||||
use crate::{
|
||||
StoreError, StoreResult, admin_revision, admin_store_error, mutation_audit,
|
||||
postgres_unavailable, require_nonempty, store_revision,
|
||||
postgres_unavailable, require_nonempty,
|
||||
};
|
||||
|
||||
use super::{ControlPlaneRepository, PgControlPlaneRepository};
|
||||
@@ -310,17 +310,12 @@ impl UpdateClientApiKeyDetails {
|
||||
|
||||
#[async_trait]
|
||||
pub trait ClientApiKeyRepository: Send + Sync {
|
||||
async fn authenticate_client_api_key(
|
||||
&self,
|
||||
key: &str,
|
||||
) -> StoreResult<Option<ClientApiKeySecret>>;
|
||||
async fn list_client_api_keys(
|
||||
&self,
|
||||
query: ClientApiKeyListQuery,
|
||||
) -> StoreResult<ClientApiKeyPage>;
|
||||
async fn get_client_api_key(&self, id: &str) -> StoreResult<Option<ClientApiKeyRecord>>;
|
||||
async fn reveal_client_api_key(&self, id: &str) -> StoreResult<Option<ClientApiKeySecret>>;
|
||||
async fn insert_client_api_key(&self, key: NewClientApiKey) -> StoreResult<()>;
|
||||
async fn update_client_api_key(&self, key: UpdateClientApiKey) -> StoreResult<bool>;
|
||||
async fn touch_client_api_keys(
|
||||
&self,
|
||||
@@ -342,22 +337,6 @@ impl PgClientApiKeyRepository {
|
||||
|
||||
#[async_trait]
|
||||
impl ClientApiKeyRepository for PgClientApiKeyRepository {
|
||||
async fn authenticate_client_api_key(
|
||||
&self,
|
||||
key: &str,
|
||||
) -> StoreResult<Option<ClientApiKeySecret>> {
|
||||
validate_key(key)?;
|
||||
let row = sqlx::query_as::<_, (String, String, String, bool, i64, i64)>(
|
||||
"select id, key, provider_kind, enabled, max_concurrency, requests_per_minute
|
||||
from client_api_keys where key = $1",
|
||||
)
|
||||
.bind(key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("authenticate client API key"))?;
|
||||
row.map(client_secret_from_row).transpose()
|
||||
}
|
||||
|
||||
async fn list_client_api_keys(
|
||||
&self,
|
||||
query: ClientApiKeyListQuery,
|
||||
@@ -432,27 +411,6 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository {
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn insert_client_api_key(&self, key: NewClientApiKey) -> StoreResult<()> {
|
||||
key.validate()?;
|
||||
sqlx::query(
|
||||
"insert into client_api_keys (
|
||||
id, name, label, provider_kind, key, enabled, max_concurrency, requests_per_minute,
|
||||
last_used_at, created_at, updated_at
|
||||
) values ($1, $2, $3, $4, $5, true, $6, $7, null, now(), now())",
|
||||
)
|
||||
.bind(key.id)
|
||||
.bind(key.name)
|
||||
.bind(key.label)
|
||||
.bind(key.provider_kind)
|
||||
.bind(key.key)
|
||||
.bind(to_i64(key.max_concurrency)?)
|
||||
.bind(to_i64(key.requests_per_minute)?)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("insert client API key"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_client_api_key(&self, key: UpdateClientApiKey) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "id", &key.id)?;
|
||||
require_nonempty(ENTITY, "name", &key.name)?;
|
||||
@@ -675,7 +633,6 @@ impl ClientKeyStore for PgAdminClientKeyStore {
|
||||
let revision = self
|
||||
.control_plane
|
||||
.create_client_api_key(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
NewClientApiKey {
|
||||
id: id.as_str().to_owned(),
|
||||
name: command.name,
|
||||
@@ -718,7 +675,6 @@ impl ClientKeyStore for PgAdminClientKeyStore {
|
||||
let revision = self
|
||||
.control_plane
|
||||
.update_client_api_key(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
UpdateClientApiKeyDetails {
|
||||
id: id.as_str().to_owned(),
|
||||
name: command.name,
|
||||
@@ -758,7 +714,6 @@ impl ClientKeyStore for PgAdminClientKeyStore {
|
||||
let revision = self
|
||||
.control_plane
|
||||
.set_client_api_key_enabled(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
id.as_str(),
|
||||
command.enabled,
|
||||
mutation_audit(
|
||||
@@ -781,7 +736,6 @@ impl ClientKeyStore for PgAdminClientKeyStore {
|
||||
) -> AdminStoreResult<gateway_admin::model::Revision> {
|
||||
self.control_plane
|
||||
.delete_client_api_key(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
command.id.as_str(),
|
||||
mutation_audit(
|
||||
context,
|
||||
|
||||
@@ -10,7 +10,7 @@ use gateway_core::accounting::{CostSource as CoreCostSource, Usage as CoreUsage}
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord as CoreAttemptRecord, ExecutionStore, IntermediateFailure,
|
||||
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest as CoreNewModelRequest, RecoveryReport as CoreRecoveryReport,
|
||||
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
|
||||
UpstreamSendState as CoreUpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
@@ -295,6 +295,12 @@ pub struct ModelRequestRecoveryReport {
|
||||
#[async_trait]
|
||||
pub trait ModelRequestRepository: Send + Sync {
|
||||
async fn insert_model_request(&self, request: NewModelRequest) -> StoreResult<()>;
|
||||
/// 请求行与首次 attempt 列一次插入;`attempt.attempt_count` 必须为 1。
|
||||
async fn insert_model_request_with_first_attempt(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
attempt: ModelRequestAttemptStart,
|
||||
) -> StoreResult<()>;
|
||||
async fn begin_model_request_attempt(
|
||||
&self,
|
||||
attempt: ModelRequestAttemptStart,
|
||||
@@ -383,11 +389,71 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn insert_model_request_with_first_attempt(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
attempt: ModelRequestAttemptStart,
|
||||
) -> StoreResult<()> {
|
||||
request.validate()?;
|
||||
attempt.validate()?;
|
||||
if attempt.attempt_count != 1 || attempt.model_request_id != request.id {
|
||||
return Err(invalid("first attempt must target the inserted request"));
|
||||
}
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_id, client_api_key_ref, config_revision, protocol,
|
||||
operation, endpoint, client_transport, requested_model_id,
|
||||
client_ip, user_agent, reasoning_effort,
|
||||
reasoning_preset, request_kind, subagent_kind, compact,
|
||||
image_generation_requested, started_at, deadline_at,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
upstream_model_id, upstream_transport, http_version,
|
||||
attempt_count, upstream_send_state
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
|
||||
$13, $14, $15, $16, $17, $18, $19,
|
||||
$20, $21, $22, $23, $24, $25, 1, 'not_sent'
|
||||
)",
|
||||
)
|
||||
.bind(request.id)
|
||||
.bind(request.client_api_key_id)
|
||||
.bind(request.client_api_key_ref)
|
||||
.bind(to_i64(request.config_revision, "config_revision")?)
|
||||
.bind(request.protocol)
|
||||
.bind(request.operation)
|
||||
.bind(request.endpoint)
|
||||
.bind(request.client_transport)
|
||||
.bind(request.requested_model_id)
|
||||
.bind(request.client_ip)
|
||||
.bind(request.user_agent)
|
||||
.bind(request.reasoning_effort)
|
||||
.bind(request.reasoning_preset)
|
||||
.bind(request.request_kind)
|
||||
.bind(request.subagent_kind)
|
||||
.bind(request.compact)
|
||||
.bind(request.image_generation_requested)
|
||||
.bind(request.started_at)
|
||||
.bind(request.deadline_at)
|
||||
.bind(attempt.provider_kind)
|
||||
.bind(attempt.provider_account_id)
|
||||
.bind(attempt.provider_account_ref)
|
||||
.bind(attempt.upstream_model_id)
|
||||
.bind(attempt.upstream_transport)
|
||||
.bind(attempt.http_version)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn begin_model_request_attempt(
|
||||
&self,
|
||||
attempt: ModelRequestAttemptStart,
|
||||
) -> StoreResult<u32> {
|
||||
attempt.validate()?;
|
||||
// upstream_send_state 是请求级单调水位(sent > ambiguous > not_sent),
|
||||
// 由 mark/finalize 抬升;开启新 attempt 不得把已持久化的 sent 重置回
|
||||
// not_sent——崩溃恢复的终态写回会原样继承本列。
|
||||
let count = sqlx::query_scalar::<_, i32>(
|
||||
"update model_requests
|
||||
set provider_kind = $2,
|
||||
@@ -396,8 +462,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
upstream_model_id = $5,
|
||||
upstream_transport = $6,
|
||||
http_version = $7,
|
||||
attempt_count = $8,
|
||||
upstream_send_state = 'not_sent'
|
||||
attempt_count = $8
|
||||
where id = $1 and outcome = 'running' and downstream_committed_at is null
|
||||
and $8 = attempt_count + 1
|
||||
returning attempt_count",
|
||||
@@ -632,53 +697,15 @@ impl ExecutionStore for PgExecutionStore {
|
||||
&self,
|
||||
request: CoreNewModelRequest,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
self.insert_model_request(NewModelRequest {
|
||||
id: request.id.as_str().to_owned(),
|
||||
client_api_key_id: request
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
client_api_key_ref: request.client_api_key_ref.as_str().to_owned(),
|
||||
config_revision: request.config_revision.get(),
|
||||
protocol: request.protocol,
|
||||
operation: request.operation.as_str().to_owned(),
|
||||
endpoint: request.endpoint,
|
||||
client_transport: request.client_transport,
|
||||
requested_model_id: request.requested_model.as_str().to_owned(),
|
||||
client_ip: request.client_ip.map(|address| address.to_string()),
|
||||
user_agent: request.user_agent,
|
||||
reasoning_effort: request.reasoning_effort,
|
||||
reasoning_preset: request.reasoning_preset,
|
||||
request_kind: request.request_kind,
|
||||
subagent_kind: request.subagent_kind,
|
||||
compact: request.compact,
|
||||
image_generation_requested: request.image_generation_requested,
|
||||
started_at: DateTime::<Utc>::from(request.started_at),
|
||||
deadline_at: DateTime::<Utc>::from(request.deadline_at),
|
||||
})
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
self.insert_model_request(new_model_request_row(request))
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn record_attempt(&self, attempt: CoreAttemptRecord) -> Result<(), CoreStoreError> {
|
||||
let expected_count = attempt.attempt_count.get();
|
||||
let persisted = self
|
||||
.begin_model_request_attempt(ModelRequestAttemptStart {
|
||||
model_request_id: attempt.request_id.as_str().to_owned(),
|
||||
attempt_count: expected_count,
|
||||
provider_kind: attempt.provider_kind.as_str().to_owned(),
|
||||
provider_account_id: attempt
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
provider_account_ref: attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
upstream_model_id: attempt.upstream_model_id.as_str().to_owned(),
|
||||
upstream_transport: attempt.upstream_transport,
|
||||
http_version: attempt.http_version,
|
||||
})
|
||||
.begin_model_request_attempt(attempt_start_row(attempt))
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
if persisted == expected_count {
|
||||
@@ -688,6 +715,24 @@ impl ExecutionStore for PgExecutionStore {
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_model_request_with_attempt(
|
||||
&self,
|
||||
request: CoreNewModelRequest,
|
||||
attempt: CoreAttemptRecord,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
// 合并写只对首次 attempt 成立(插入即带 attempt 列);其余走基础两步。
|
||||
if attempt.attempt_count.get() != 1 {
|
||||
self.create_model_request(request).await?;
|
||||
return self.record_attempt(attempt).await;
|
||||
}
|
||||
self.insert_model_request_with_first_attempt(
|
||||
new_model_request_row(request),
|
||||
attempt_start_row(attempt),
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
@@ -776,6 +821,56 @@ impl ExecutionStore for PgExecutionStore {
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), CoreStoreError> {
|
||||
let error = failure.error;
|
||||
let retry_after_ms = error
|
||||
.retry_after()
|
||||
.map(|duration| u64::try_from(duration.as_millis()))
|
||||
.transpose()
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let latency_ms = u64::try_from(failure.latency.as_millis())
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = failure.provider_kind.as_str(),
|
||||
account_id = failure.account_id.as_str(),
|
||||
upstream_model = failure.upstream_model_id.as_str(),
|
||||
failure_kind = error.kind().as_str(),
|
||||
send_state = ?error.send_state(),
|
||||
upstream_status = ?error.upstream_status(),
|
||||
provider_error_code = ?provider_error_code,
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
super::OpsEventRepository::append_ops_event(
|
||||
&super::PgOpsEventRepository::new(self.pool.clone()),
|
||||
super::OpsEvent {
|
||||
id: Uuid::now_v7().to_string(),
|
||||
model_request_id: None,
|
||||
attempt_index: None,
|
||||
level: super::OpsEventLevel::Warning,
|
||||
component: "account_probe".to_owned(),
|
||||
operation: "connection_test".to_owned(),
|
||||
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
||||
provider_account_id: Some(failure.account_id.as_str().to_owned()),
|
||||
provider_account_ref: Some(failure.account_id.as_str().to_owned()),
|
||||
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
|
||||
failure_kind: error.kind().as_str().to_owned(),
|
||||
status_code: error.upstream_status(),
|
||||
provider_error_code,
|
||||
retry_after_ms,
|
||||
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
|
||||
latency_ms: Some(latency_ms),
|
||||
message: "account connection test failed".to_owned(),
|
||||
occurrence_count: 1,
|
||||
created_at: Utc::now(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: CoreModelRequestFinalization,
|
||||
@@ -937,6 +1032,52 @@ fn core_store_error(error: StoreError) -> CoreStoreError {
|
||||
CoreStoreError::new(kind)
|
||||
}
|
||||
|
||||
fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
|
||||
NewModelRequest {
|
||||
id: request.id.as_str().to_owned(),
|
||||
client_api_key_id: request
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
client_api_key_ref: request.client_api_key_ref.as_str().to_owned(),
|
||||
config_revision: request.config_revision.get(),
|
||||
protocol: request.protocol,
|
||||
operation: request.operation.as_str().to_owned(),
|
||||
endpoint: request.endpoint,
|
||||
client_transport: request.client_transport,
|
||||
requested_model_id: request.requested_model.as_str().to_owned(),
|
||||
client_ip: request.client_ip.map(|address| address.to_string()),
|
||||
user_agent: request.user_agent,
|
||||
reasoning_effort: request.reasoning_effort,
|
||||
reasoning_preset: request.reasoning_preset,
|
||||
request_kind: request.request_kind,
|
||||
subagent_kind: request.subagent_kind,
|
||||
compact: request.compact,
|
||||
image_generation_requested: request.image_generation_requested,
|
||||
started_at: DateTime::<Utc>::from(request.started_at),
|
||||
deadline_at: DateTime::<Utc>::from(request.deadline_at),
|
||||
}
|
||||
}
|
||||
|
||||
fn attempt_start_row(attempt: CoreAttemptRecord) -> ModelRequestAttemptStart {
|
||||
ModelRequestAttemptStart {
|
||||
model_request_id: attempt.request_id.as_str().to_owned(),
|
||||
attempt_count: attempt.attempt_count.get(),
|
||||
provider_kind: attempt.provider_kind.as_str().to_owned(),
|
||||
provider_account_id: attempt
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
provider_account_ref: attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
upstream_model_id: attempt.upstream_model_id.as_str().to_owned(),
|
||||
upstream_transport: attempt.upstream_transport,
|
||||
http_version: attempt.http_version,
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_i64(value: Option<u64>, field: &'static str) -> StoreResult<Option<i64>> {
|
||||
value.map(|value| to_i64(value, field)).transpose()
|
||||
}
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
//! 不含正文或会话状态的模型请求历史查询。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::{StoreResult, postgres_unavailable, require_nonempty};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ModelRequestHistoryRecord {
|
||||
pub id: String,
|
||||
pub client_api_key_ref: String,
|
||||
pub requested_model_id: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub outcome: String,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_response_id: Option<String>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ModelRequestHistoryRepository: Send + Sync {
|
||||
async fn find_model_request_history(
|
||||
&self,
|
||||
model_request_id: &str,
|
||||
) -> StoreResult<Option<ModelRequestHistoryRecord>>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PgHistoryRepository {
|
||||
pool: PgPool,
|
||||
}
|
||||
|
||||
impl PgHistoryRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ModelRequestHistoryRepository for PgHistoryRepository {
|
||||
async fn find_model_request_history(
|
||||
&self,
|
||||
model_request_id: &str,
|
||||
) -> StoreResult<Option<ModelRequestHistoryRecord>> {
|
||||
require_nonempty(
|
||||
"model request history",
|
||||
"model_request_id",
|
||||
model_request_id,
|
||||
)?;
|
||||
fetch_history(&self.pool, "id", model_request_id).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_history(
|
||||
pool: &PgPool,
|
||||
lookup: &'static str,
|
||||
value: &str,
|
||||
) -> StoreResult<Option<ModelRequestHistoryRecord>> {
|
||||
let row = match lookup {
|
||||
"id" => {
|
||||
sqlx::query_as::<_, HistoryRow>(
|
||||
"select id, client_api_key_ref, requested_model_id, provider_kind,
|
||||
provider_account_ref, upstream_model_id, outcome,
|
||||
client_response_id, upstream_response_id,
|
||||
started_at, completed_at
|
||||
from model_requests where id = $1",
|
||||
)
|
||||
.bind(value)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
_ => return Ok(None),
|
||||
}
|
||||
.map_err(|_| postgres_unavailable("read model request history"))?;
|
||||
Ok(row.map(history_from_row))
|
||||
}
|
||||
|
||||
type HistoryRow = (
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
DateTime<Utc>,
|
||||
Option<DateTime<Utc>>,
|
||||
);
|
||||
|
||||
fn history_from_row(row: HistoryRow) -> ModelRequestHistoryRecord {
|
||||
ModelRequestHistoryRecord {
|
||||
id: row.0,
|
||||
client_api_key_ref: row.1,
|
||||
requested_model_id: row.2,
|
||||
provider_kind: row.3,
|
||||
provider_account_ref: row.4,
|
||||
upstream_model_id: row.5,
|
||||
outcome: row.6,
|
||||
client_response_id: row.7,
|
||||
upstream_response_id: row.8,
|
||||
started_at: row.9,
|
||||
completed_at: row.10,
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,14 @@
|
||||
use async_trait::async_trait;
|
||||
use sqlx::{PgPool, postgres::PgPoolOptions};
|
||||
|
||||
use crate::{Revision, StoreBackend, StoreError, StoreResult, postgres_unavailable};
|
||||
use crate::{
|
||||
Revision, StoreBackend, StoreError, StorePoolConfig, StoreResult, postgres_unavailable,
|
||||
};
|
||||
|
||||
mod admin_security_audit;
|
||||
mod admission_recovery;
|
||||
mod client_keys;
|
||||
mod execution;
|
||||
mod history;
|
||||
mod observability;
|
||||
mod ops_events;
|
||||
mod provider_accounts;
|
||||
@@ -21,7 +22,6 @@ pub use admin_security_audit::*;
|
||||
pub use admission_recovery::*;
|
||||
pub use client_keys::*;
|
||||
pub use execution::*;
|
||||
pub use history::*;
|
||||
pub use observability::*;
|
||||
pub use ops_events::*;
|
||||
pub use provider_accounts::*;
|
||||
@@ -31,12 +31,19 @@ pub use snapshot::*;
|
||||
|
||||
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
|
||||
|
||||
/// 建立 PostgreSQL pool 并只执行冻结的 `0001_initial.sql` migration 集。
|
||||
pub async fn connect_and_migrate(database_url: &str) -> StoreResult<PgPool> {
|
||||
/// 建立 PostgreSQL pool 并只执行冻结的 migration 集。
|
||||
pub async fn connect_and_migrate(
|
||||
database_url: &str,
|
||||
pool_config: StorePoolConfig,
|
||||
) -> StoreResult<PgPool> {
|
||||
if database_url.trim().is_empty() {
|
||||
return Err(postgres_unavailable("connect PostgreSQL"));
|
||||
}
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(pool_config.max_connections)
|
||||
.acquire_timeout(std::time::Duration::from_secs(
|
||||
pool_config.acquire_timeout_seconds,
|
||||
))
|
||||
.connect(database_url)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("connect PostgreSQL"))?;
|
||||
@@ -67,27 +74,23 @@ pub trait ControlPlaneRepository: Send + Sync {
|
||||
|
||||
async fn replace_control_plane(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
replacement: ControlPlaneReplacement,
|
||||
) -> StoreResult<ControlPlaneSnapshot>;
|
||||
|
||||
async fn create_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
key: NewClientApiKey,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision>;
|
||||
|
||||
async fn update_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
key: UpdateClientApiKeyDetails,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision>;
|
||||
|
||||
async fn set_client_api_key_enabled(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
id: &str,
|
||||
enabled: bool,
|
||||
audit: AdminAuditEvent,
|
||||
@@ -95,7 +98,6 @@ pub trait ControlPlaneRepository: Send + Sync {
|
||||
|
||||
async fn delete_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
id: &str,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision>;
|
||||
@@ -135,7 +137,6 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
|
||||
|
||||
async fn replace_control_plane(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
replacement: ControlPlaneReplacement,
|
||||
) -> StoreResult<ControlPlaneSnapshot> {
|
||||
replacement.settings.validate()?;
|
||||
@@ -145,12 +146,9 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin control plane replacement"))?;
|
||||
let result = async {
|
||||
let revision = update_runtime_settings_in_transaction(
|
||||
&mut transaction,
|
||||
expected_revision,
|
||||
&replacement.settings,
|
||||
)
|
||||
.await?;
|
||||
let revision =
|
||||
update_runtime_settings_in_transaction(&mut transaction, &replacement.settings)
|
||||
.await?;
|
||||
append_admin_audit_event_in_transaction(&mut transaction, replacement.audit, revision)
|
||||
.await?;
|
||||
load_control_plane_in_transaction(&mut transaction).await
|
||||
@@ -176,41 +174,29 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
|
||||
|
||||
async fn create_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
key: NewClientApiKey,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
self.apply_targeted_mutation(
|
||||
expected_revision,
|
||||
ControlPlaneMutation::CreateClientApiKey(key),
|
||||
audit,
|
||||
)
|
||||
.await
|
||||
self.apply_targeted_mutation(ControlPlaneMutation::CreateClientApiKey(key), audit)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
key: UpdateClientApiKeyDetails,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
self.apply_targeted_mutation(
|
||||
expected_revision,
|
||||
ControlPlaneMutation::UpdateClientApiKey(key),
|
||||
audit,
|
||||
)
|
||||
.await
|
||||
self.apply_targeted_mutation(ControlPlaneMutation::UpdateClientApiKey(key), audit)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn set_client_api_key_enabled(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
id: &str,
|
||||
enabled: bool,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
self.apply_targeted_mutation(
|
||||
expected_revision,
|
||||
ControlPlaneMutation::SetClientApiKeyEnabled {
|
||||
id: id.to_owned(),
|
||||
enabled,
|
||||
@@ -222,12 +208,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
|
||||
|
||||
async fn delete_client_api_key(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
id: &str,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
self.apply_targeted_mutation(
|
||||
expected_revision,
|
||||
ControlPlaneMutation::DeleteClientApiKey(id.to_owned()),
|
||||
audit,
|
||||
)
|
||||
@@ -245,7 +229,6 @@ enum ControlPlaneMutation {
|
||||
impl PgControlPlaneRepository {
|
||||
async fn apply_targeted_mutation(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
mutation: ControlPlaneMutation,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
@@ -255,8 +238,7 @@ impl PgControlPlaneRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin targeted control plane mutation"))?;
|
||||
let result = async {
|
||||
let revision =
|
||||
bump_config_revision_in_transaction(&mut transaction, expected_revision).await?;
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
match mutation {
|
||||
ControlPlaneMutation::CreateClientApiKey(key) => {
|
||||
insert_client_api_key_in_transaction(&mut transaction, &key).await?;
|
||||
|
||||
@@ -23,7 +23,8 @@ const MAX_PAGE_SIZE: u16 = 100;
|
||||
const MAX_FILTER_BYTES: usize = 256;
|
||||
const MAX_SEARCH_BYTES: usize = 512;
|
||||
const MAX_ACCOUNT_IDS: usize = 200;
|
||||
const DASHBOARD_ACCOUNT_LIMIT: u16 = 50;
|
||||
/// 概览卡只展示最近使用的四个账号;完整账号用量由账号管理页单独查询。
|
||||
const DASHBOARD_ACCOUNT_LIMIT: u16 = 4;
|
||||
const DIAGNOSTIC_LIMIT: i64 = 100;
|
||||
const ACCOUNT_USAGE_TIMELINE_HOURS: i64 = 24;
|
||||
|
||||
@@ -660,7 +661,8 @@ pub struct OpsErrorRecord {
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
pub status_code: Option<u16>,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
@@ -757,35 +759,35 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<DashboardObservation> {
|
||||
let filter = UsageRecordFilter::default();
|
||||
let requests = request_metrics(&self.pool, range, &filter).await?;
|
||||
let attempts = attempt_metrics(&self.pool, range, &filter).await?;
|
||||
let provider_accounts = provider_account_metrics(&self.pool, range.end).await?;
|
||||
let trend = request_metric_series(&self.pool, range, &filter).await?;
|
||||
let account_usage_range = ObservabilityRange::new(
|
||||
range.end - TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS),
|
||||
range.end,
|
||||
)?;
|
||||
let account_usage = provider_account_usage(
|
||||
&self.pool,
|
||||
let account_usage_query =
|
||||
ProviderAccountUsageQuery::recent(account_usage_range, DASHBOARD_ACCOUNT_LIMIT)?
|
||||
.with_hourly_request_buckets()?,
|
||||
)
|
||||
.await?;
|
||||
let recent_requests = list_usage_records(
|
||||
&self.pool,
|
||||
UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
outcome: Some("succeeded".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1)?,
|
||||
page_size: ObservabilityPageSize::new(10)?,
|
||||
.with_hourly_request_buckets()?;
|
||||
let recent_query = UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
outcome: Some("succeeded".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.items;
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1)?,
|
||||
page_size: ObservabilityPageSize::new(10)?,
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, provider_accounts) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
provider_account_metrics(&self.pool, range.end),
|
||||
)?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
async { Ok(list_usage_records(&self.pool, recent_query).await?.items) },
|
||||
)?;
|
||||
Ok(DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
@@ -1626,7 +1628,8 @@ fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError {
|
||||
upstream_model_id: error.upstream_model_id,
|
||||
upstream_transport: error.upstream_transport,
|
||||
failure_kind: error.failure_kind,
|
||||
status_code: error.status_code,
|
||||
client_status_code: error.client_status_code,
|
||||
upstream_status_code: error.upstream_status_code,
|
||||
provider_error_code: error.provider_error_code,
|
||||
client_response_id: error.client_response_id,
|
||||
upstream_request_id: error.upstream_request_id,
|
||||
@@ -1648,7 +1651,10 @@ async fn request_metrics(
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<RequestMetrics> {
|
||||
filter.validate()?;
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
// 结果计数覆盖范围内全部请求(成功率/失败率的分母分子);用量、缓存、
|
||||
// 延迟与成本聚合仅统计用量事实(已完整交付客户端的成功响应)。
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
@@ -1656,40 +1662,43 @@ async fn request_metrics(
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens), 0)::bigint as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens), 0)::bigint as reasoning_tokens,
|
||||
coalesce(sum(total_tokens), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms), 0)::bigint as first_token_latency_sum,
|
||||
count(first_token_ms)::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms), 0)::bigint as latency_sum,
|
||||
count(latency_ms)::bigint as latency_count,
|
||||
max(latency_ms)::bigint as max_latency_ms,
|
||||
min(latency_ms)::bigint as min_latency_ms,
|
||||
count(*) filter (where input_tokens is not null)::bigint
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where input_tokens is not null and cached_tokens > 0)::bigint
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
as latency_p50_ms,
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
as latency_p95_ms,
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
as latency_p99_ms,
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
as first_token_p50_ms,
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
as first_token_p95_ms,
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
as first_token_p99_ms
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
filter (where {fact}) as first_token_p99_ms
|
||||
from model_requests mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
let row = query
|
||||
.build()
|
||||
@@ -1706,9 +1715,12 @@ async fn request_metric_series(
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
filter.validate()?;
|
||||
let granularity = granularity_for(range);
|
||||
// 与 request_metrics 同一契约:结果计数覆盖全部请求,用量/延迟/成本
|
||||
// 聚合仅统计用量事实。
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
|
||||
query.push_bind(granularity.sql_interval());
|
||||
query.push(
|
||||
query.push(format!(
|
||||
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
|
||||
count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
@@ -1717,46 +1729,49 @@ async fn request_metric_series(
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens), 0)::bigint as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens), 0)::bigint as reasoning_tokens,
|
||||
coalesce(sum(total_tokens), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms), 0)::bigint as first_token_latency_sum,
|
||||
count(first_token_ms)::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms), 0)::bigint as latency_sum,
|
||||
count(latency_ms)::bigint as latency_count,
|
||||
max(latency_ms)::bigint as max_latency_ms,
|
||||
min(latency_ms)::bigint as min_latency_ms,
|
||||
count(*) filter (where input_tokens is not null)::bigint
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where input_tokens is not null and cached_tokens > 0)::bigint
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
as latency_p50_ms,
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
as latency_p95_ms,
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
as latency_p99_ms,
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
as first_token_p50_ms,
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
as first_token_p95_ms,
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
as first_token_p99_ms,
|
||||
count(*) filter (where cost_source = 'provider_reported')::bigint
|
||||
filter (where {fact}) as first_token_p99_ms,
|
||||
count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (where cost_source = 'calculated')::bigint
|
||||
count(*) filter (where {fact} and cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(*) filter (where cost_source = 'unavailable')::bigint
|
||||
count(*) filter (where {fact} and cost_source = 'unavailable')::bigint
|
||||
as unavailable_count
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
from model_requests mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start order by bucket_start");
|
||||
let rows = query
|
||||
@@ -1977,8 +1992,12 @@ async fn provider_account_metrics(
|
||||
let row = sqlx::query(
|
||||
"select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where not enabled or availability <> 'ready')::bigint
|
||||
as unavailable,
|
||||
count(*) filter (
|
||||
where not (
|
||||
enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
)
|
||||
)::bigint as unavailable,
|
||||
count(*) filter (
|
||||
where enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
@@ -2242,8 +2261,14 @@ fn push_usage_filter(query: &mut QueryBuilder<Postgres>, filter: &UsageRecordFil
|
||||
/// `model_requests` 同时承担执行审计:包括上游已发送、但尚未收到首个事件就断开的
|
||||
/// WebSocket 请求。这些失败仍必须留给 Ops Errors 和调度健康度分析,不能混入用量、
|
||||
/// 成本、账号使用次数或请求明细。
|
||||
fn completed_usage_fact_predicate(alias: &str) -> String {
|
||||
format!(
|
||||
"{alias}.outcome = 'succeeded' and {alias}.downstream_committed_at is not null and {alias}.client_status_code between 200 and 399"
|
||||
)
|
||||
}
|
||||
|
||||
fn push_completed_usage_fact_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
|
||||
query.push(format!(" and {alias}.outcome = 'succeeded' and {alias}.downstream_committed_at is not null and {alias}.client_status_code between 200 and 399"));
|
||||
query.push(format!(" and {}", completed_usage_fact_predicate(alias)));
|
||||
}
|
||||
|
||||
async fn provider_account_usage(
|
||||
@@ -2921,7 +2946,7 @@ const OPS_ERRORS_CTE: &str = "with errors as (
|
||||
mr.provider_kind,
|
||||
mr.provider_account_ref, mr.upstream_model_id, mr.upstream_transport,
|
||||
coalesce(mr.error_kind, 'failed') as failure_kind,
|
||||
coalesce(mr.upstream_status_code, mr.client_status_code) as status_code,
|
||||
mr.client_status_code, mr.upstream_status_code,
|
||||
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
|
||||
mr.latency_ms, coalesce(mr.error_message, mr.error_kind, 'request failed') as message,
|
||||
1::integer as occurrence_count,
|
||||
@@ -2932,7 +2957,8 @@ const OPS_ERRORS_CTE: &str = "with errors as (
|
||||
select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index,
|
||||
mr.client_api_key_ref, oe.component, oe.operation,
|
||||
oe.provider_kind, oe.provider_account_ref,
|
||||
oe.upstream_model_id, null::text, oe.failure_kind, oe.status_code,
|
||||
oe.upstream_model_id, null::text, oe.failure_kind,
|
||||
null::integer as client_status_code, oe.status_code as upstream_status_code,
|
||||
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
|
||||
oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at,
|
||||
'ops_event:' || oe.id
|
||||
@@ -2947,7 +2973,8 @@ async fn list_ops_errors(pool: &PgPool, query: OpsErrorQuery) -> StoreResult<Ops
|
||||
" select source, event_id, request_id, attempt_index, client_api_key_ref,
|
||||
component, operation, provider_kind, provider_account_ref,
|
||||
upstream_model_id, upstream_transport,
|
||||
failure_kind, status_code, provider_error_code, client_response_id,
|
||||
failure_kind, client_status_code, upstream_status_code,
|
||||
provider_error_code, client_response_id,
|
||||
upstream_request_id, latency_ms, message, occurrence_count, occurred_at,
|
||||
stable_sort_id
|
||||
from errors e where e.occurred_at >= ",
|
||||
@@ -3046,7 +3073,7 @@ fn push_ops_filter(statement: &mut QueryBuilder<Postgres>, filter: &OpsErrorFilt
|
||||
statement.push_bind(i32::try_from(index).unwrap_or(i32::MAX));
|
||||
}
|
||||
if let Some(status) = filter.status_code {
|
||||
statement.push(" and e.status_code = ");
|
||||
statement.push(" and e.upstream_status_code = ");
|
||||
statement.push_bind(i32::from(status));
|
||||
}
|
||||
if let Some(search) = &filter.search {
|
||||
@@ -3147,7 +3174,8 @@ fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<OpsErrorRecord
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
upstream_transport: get(row, "upstream_transport")?,
|
||||
failure_kind: get(row, "failure_kind")?,
|
||||
status_code: optional_status(row, "status_code")?,
|
||||
client_status_code: optional_status(row, "client_status_code")?,
|
||||
upstream_status_code: optional_status(row, "upstream_status_code")?,
|
||||
provider_error_code: get(row, "provider_error_code")?,
|
||||
client_response_id: get(row, "client_response_id")?,
|
||||
upstream_request_id: get(row, "upstream_request_id")?,
|
||||
|
||||
@@ -460,10 +460,6 @@ pub trait ProviderAccountRepository: Send + Sync {
|
||||
&self,
|
||||
update: ProviderCredentialUpdate,
|
||||
) -> StoreResult<Revision>;
|
||||
async fn update_provider_account_observation(
|
||||
&self,
|
||||
observation: ProviderAccountObservation,
|
||||
) -> StoreResult<bool>;
|
||||
async fn apply_provider_account_state(
|
||||
&self,
|
||||
update: ProviderAccountStateUpdate,
|
||||
@@ -489,25 +485,21 @@ pub trait ProviderAccountAdminRepository: Send + Sync {
|
||||
|
||||
async fn import_provider_accounts(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: ImportProviderAccounts,
|
||||
) -> StoreResult<ProviderAccountAdminImport>;
|
||||
|
||||
async fn rotate_provider_account(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: RotateProviderAccount,
|
||||
) -> StoreResult<ProviderAccountAdminRotation>;
|
||||
|
||||
async fn set_provider_account_enabled_admin(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: SetProviderAccountEnabled,
|
||||
) -> StoreResult<Revision>;
|
||||
|
||||
async fn delete_provider_accounts_admin(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: DeleteProviderAccounts,
|
||||
) -> StoreResult<Revision>;
|
||||
}
|
||||
@@ -654,36 +646,6 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
Revision::new(to_u64(next)?)
|
||||
}
|
||||
|
||||
async fn update_provider_account_observation(
|
||||
&self,
|
||||
observation: ProviderAccountObservation,
|
||||
) -> StoreResult<bool> {
|
||||
observation.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = $2, availability_reason = $3, cooldown_until = $4,
|
||||
provider_quota_json = $5, availability_observed_at = $6,
|
||||
quota_observed_at = $7,
|
||||
updated_at = greatest(now(), $6, coalesce($7, $6))
|
||||
where id = $1",
|
||||
)
|
||||
.bind(observation.account_id)
|
||||
.bind(observation.availability.as_str())
|
||||
.bind(observation.availability_reason)
|
||||
.bind(observation.cooldown_until)
|
||||
.bind(
|
||||
observation
|
||||
.provider_quota_json
|
||||
.map(|quota| quota.as_value()),
|
||||
)
|
||||
.bind(observation.availability_observed_at)
|
||||
.bind(observation.quota_observed_at)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("update provider account observation"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn apply_provider_account_state(
|
||||
&self,
|
||||
update: ProviderAccountStateUpdate,
|
||||
@@ -801,7 +763,6 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn import_provider_accounts(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: ImportProviderAccounts,
|
||||
) -> StoreResult<ProviderAccountAdminImport> {
|
||||
command.validate()?;
|
||||
@@ -811,9 +772,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin provider account admin import"))?;
|
||||
let result = async {
|
||||
let revision =
|
||||
bump_config_revision_in_transaction(&mut transaction, expected_config_revision)
|
||||
.await?;
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
let mut account_ids = Vec::with_capacity(command.accounts.len());
|
||||
for account in &command.accounts {
|
||||
account_ids
|
||||
@@ -832,7 +791,6 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn rotate_provider_account(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: RotateProviderAccount,
|
||||
) -> StoreResult<ProviderAccountAdminRotation> {
|
||||
command.scope.validate()?;
|
||||
@@ -848,9 +806,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin provider account admin rotation"))?;
|
||||
let result = async {
|
||||
let config_revision =
|
||||
bump_config_revision_in_transaction(&mut transaction, expected_config_revision)
|
||||
.await?;
|
||||
let config_revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
let credential_revision = rotate_provider_account_in_transaction(
|
||||
&mut transaction,
|
||||
&command.scope,
|
||||
@@ -875,7 +831,6 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn set_provider_account_enabled_admin(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: SetProviderAccountEnabled,
|
||||
) -> StoreResult<Revision> {
|
||||
command.scope.validate()?;
|
||||
@@ -886,9 +841,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin provider account admin state change"))?;
|
||||
let result = async {
|
||||
let revision =
|
||||
bump_config_revision_in_transaction(&mut transaction, expected_config_revision)
|
||||
.await?;
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
set_provider_account_enabled_in_transaction(
|
||||
&mut transaction,
|
||||
&command.scope,
|
||||
@@ -906,7 +859,6 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn delete_provider_accounts_admin(
|
||||
&self,
|
||||
expected_config_revision: Revision,
|
||||
command: DeleteProviderAccounts,
|
||||
) -> StoreResult<Revision> {
|
||||
command.scope.validate()?;
|
||||
@@ -917,9 +869,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin provider account admin deletion"))?;
|
||||
let result = async {
|
||||
let revision =
|
||||
bump_config_revision_in_transaction(&mut transaction, expected_config_revision)
|
||||
.await?;
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
delete_provider_accounts_in_transaction(
|
||||
&mut transaction,
|
||||
&command.scope,
|
||||
@@ -1066,7 +1016,6 @@ impl PgAdminAccountStore {
|
||||
|
||||
async fn commit_prepared_import(
|
||||
&self,
|
||||
expected_config_revision: AdminRevision,
|
||||
prepared: PreparedCredentialImport,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
@@ -1080,22 +1029,19 @@ impl PgAdminAccountStore {
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let imported = self
|
||||
.accounts
|
||||
.import_provider_accounts(
|
||||
store_revision(expected_config_revision)?,
|
||||
ImportProviderAccounts {
|
||||
scope: ProviderAccountAdminScope {
|
||||
provider_kind: provider_kind.clone(),
|
||||
},
|
||||
accounts,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
&provider_kind,
|
||||
vec!["credentials".to_owned()],
|
||||
),
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: ProviderAccountAdminScope {
|
||||
provider_kind: provider_kind.clone(),
|
||||
},
|
||||
)
|
||||
accounts,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
&provider_kind,
|
||||
vec!["credentials".to_owned()],
|
||||
),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
Ok(CredentialImportResult {
|
||||
@@ -1117,7 +1063,6 @@ impl PgAdminAccountStore {
|
||||
|
||||
async fn commit_prepared_rotation(
|
||||
&self,
|
||||
expected_config_revision: AdminRevision,
|
||||
prepared: PreparedCredentialRotationFacts,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
@@ -1128,36 +1073,31 @@ impl PgAdminAccountStore {
|
||||
};
|
||||
let rotation = self
|
||||
.accounts
|
||||
.rotate_provider_account(
|
||||
store_revision(expected_config_revision)?,
|
||||
RotateProviderAccount {
|
||||
scope,
|
||||
profile: UpdateProviderAccount {
|
||||
id: account_id.as_str().to_owned(),
|
||||
name: prepared.name,
|
||||
email: prepared.email,
|
||||
plan_type: prepared.plan_type,
|
||||
},
|
||||
credential: ProviderCredentialUpdate {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
expected_revision: store_revision(prepared.expected_credential_revision)?,
|
||||
provider_credentials_json: provider_document_json(
|
||||
prepared.provider_material,
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?,
|
||||
has_refresh_token: prepared.has_refresh_token,
|
||||
access_token_expires_at: prepared.access_token_expires_at,
|
||||
next_refresh_at: prepared.next_refresh_at,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
account_id.as_str(),
|
||||
vec!["credentials".to_owned()],
|
||||
),
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope,
|
||||
profile: UpdateProviderAccount {
|
||||
id: account_id.as_str().to_owned(),
|
||||
name: prepared.name,
|
||||
email: prepared.email,
|
||||
plan_type: prepared.plan_type,
|
||||
},
|
||||
)
|
||||
credential: ProviderCredentialUpdate {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
expected_revision: store_revision(prepared.expected_credential_revision)?,
|
||||
provider_credentials_json: provider_document_json(prepared.provider_material)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?,
|
||||
has_refresh_token: prepared.has_refresh_token,
|
||||
access_token_expires_at: prepared.access_token_expires_at,
|
||||
next_refresh_at: prepared.next_refresh_at,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
account_id.as_str(),
|
||||
vec!["credentials".to_owned()],
|
||||
),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
Ok(CredentialMutationResult {
|
||||
@@ -1417,13 +1357,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: CredentialImportCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialImportResult> {
|
||||
self.commit_prepared_import(
|
||||
command.expected_config_revision,
|
||||
command.prepared,
|
||||
context,
|
||||
"import_document",
|
||||
)
|
||||
.await
|
||||
self.commit_prepared_import(command.prepared, context, "import_document")
|
||||
.await
|
||||
}
|
||||
|
||||
async fn commit_authorization(
|
||||
@@ -1431,13 +1366,11 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: AuthorizationCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
let expected_config_revision = command.pending.expected_config_revision();
|
||||
match command.credential {
|
||||
AuthorizationCredentialCommit::Create(credential) => {
|
||||
let account_id = credential.account_id.clone();
|
||||
let result = self
|
||||
.commit_prepared_import(
|
||||
expected_config_revision,
|
||||
PreparedCredentialImport {
|
||||
provider_kind: credential.provider_kind.clone(),
|
||||
credentials: vec![credential],
|
||||
@@ -1465,13 +1398,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
})
|
||||
}
|
||||
AuthorizationCredentialCommit::Reauthorize(prepared) => {
|
||||
self.commit_prepared_rotation(
|
||||
expected_config_revision,
|
||||
prepared,
|
||||
context,
|
||||
"reauthorize",
|
||||
)
|
||||
.await
|
||||
self.commit_prepared_rotation(prepared, context, "reauthorize")
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1481,13 +1409,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: CredentialRotationCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.commit_prepared_rotation(
|
||||
command.expected_config_revision,
|
||||
command.prepared,
|
||||
context,
|
||||
"rotate_credential",
|
||||
)
|
||||
.await
|
||||
self.commit_prepared_rotation(command.prepared, context, "rotate_credential")
|
||||
.await
|
||||
}
|
||||
|
||||
async fn commit_credential_refresh(
|
||||
@@ -1495,13 +1418,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
command: CredentialRotationCommit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult> {
|
||||
self.commit_prepared_rotation(
|
||||
command.expected_config_revision,
|
||||
command.prepared,
|
||||
context,
|
||||
"refresh_credential",
|
||||
)
|
||||
.await
|
||||
self.commit_prepared_rotation(command.prepared, context, "refresh_credential")
|
||||
.await
|
||||
}
|
||||
|
||||
async fn set_account_enabled(
|
||||
@@ -1512,21 +1430,18 @@ impl AccountStore for PgAdminAccountStore {
|
||||
let scope = self.required_scope(&command.account_id).await?;
|
||||
let action = if command.enabled { "enable" } else { "disable" };
|
||||
self.accounts
|
||||
.set_provider_account_enabled_admin(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
SetProviderAccountEnabled {
|
||||
scope,
|
||||
account_id: command.account_id.clone(),
|
||||
enabled: command.enabled,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
&command.account_id,
|
||||
vec!["enabled".to_owned()],
|
||||
),
|
||||
},
|
||||
)
|
||||
.set_provider_account_enabled_admin(SetProviderAccountEnabled {
|
||||
scope,
|
||||
account_id: command.account_id.clone(),
|
||||
enabled: command.enabled,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
action,
|
||||
"provider_account",
|
||||
&command.account_id,
|
||||
vec!["enabled".to_owned()],
|
||||
),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))
|
||||
.and_then(admin_revision)
|
||||
@@ -1551,20 +1466,17 @@ impl AccountStore for PgAdminAccountStore {
|
||||
"provider_accounts".to_owned()
|
||||
};
|
||||
self.accounts
|
||||
.delete_provider_accounts_admin(
|
||||
store_revision(command.expected_config_revision)?,
|
||||
DeleteProviderAccounts {
|
||||
scope,
|
||||
account_ids: command.account_ids,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
"delete",
|
||||
"provider_account",
|
||||
&audit_target,
|
||||
Vec::new(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.delete_provider_accounts_admin(DeleteProviderAccounts {
|
||||
scope,
|
||||
account_ids: command.account_ids,
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
"delete",
|
||||
"provider_account",
|
||||
&audit_target,
|
||||
Vec::new(),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))
|
||||
.and_then(admin_revision)
|
||||
@@ -1695,15 +1607,14 @@ fn admin_account_status(
|
||||
} else if account.availability == ProviderAccountAvailability::QuotaExhausted {
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == ProviderAccountAvailability::Expired
|
||||
|| account.availability == ProviderAccountAvailability::Invalid
|
||||
|| account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AdminAccountStatus::Expired
|
||||
} else if account.availability == ProviderAccountAvailability::Ready {
|
||||
AdminAccountStatus::Active
|
||||
} else {
|
||||
AdminAccountStatus::Attention
|
||||
AdminAccountStatus::Active
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1728,7 +1639,7 @@ fn admin_account_summary(
|
||||
.count(),
|
||||
)
|
||||
.unwrap_or(u64::MAX);
|
||||
let attention = u64::try_from(
|
||||
let unavailable = u64::try_from(
|
||||
accounts
|
||||
.iter()
|
||||
.filter(|account| {
|
||||
@@ -1746,7 +1657,7 @@ fn admin_account_summary(
|
||||
total,
|
||||
active,
|
||||
quota_exhausted,
|
||||
attention,
|
||||
unavailable,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1797,7 +1708,6 @@ const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
|
||||
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
AdminAccountStatus::Disabled => "disabled",
|
||||
AdminAccountStatus::Banned => "banned",
|
||||
AdminAccountStatus::Attention => "attention",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,47 +78,49 @@ impl RetentionRepository for PgRetentionRepository {
|
||||
});
|
||||
}
|
||||
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin retention transaction"))?;
|
||||
let model_requests = sqlx::query(
|
||||
// 分批删除,每批独立提交:清理是幂等的,无需跨表原子性;
|
||||
// 有界批量避免百万行单事务的长锁与 WAL 峰值。
|
||||
let model_requests = purge_in_batches(
|
||||
&self.pool,
|
||||
"delete from model_requests
|
||||
where outcome <> 'running'
|
||||
and completed_at < $1 - ($2 * interval '1 day')",
|
||||
where ctid in (
|
||||
select ctid from model_requests
|
||||
where outcome <> 'running'
|
||||
and completed_at < $1 - ($2 * interval '1 day')
|
||||
limit $3
|
||||
)",
|
||||
now,
|
||||
settings.usage_retention_days,
|
||||
"delete expired model requests",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(i64::from(settings.usage_retention_days))
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("delete expired model requests"))?
|
||||
.rows_affected();
|
||||
let ops_events = sqlx::query(
|
||||
.await?;
|
||||
let ops_events = purge_in_batches(
|
||||
&self.pool,
|
||||
"delete from ops_events
|
||||
where model_request_id is null
|
||||
and created_at < $1 - ($2 * interval '1 day')",
|
||||
where ctid in (
|
||||
select ctid from ops_events
|
||||
where model_request_id is null
|
||||
and created_at < $1 - ($2 * interval '1 day')
|
||||
limit $3
|
||||
)",
|
||||
now,
|
||||
settings.ops_event_retention_days,
|
||||
"delete expired ops events",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(i64::from(settings.ops_event_retention_days))
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("delete expired ops events"))?
|
||||
.rows_affected();
|
||||
let admin_audit_events = sqlx::query(
|
||||
.await?;
|
||||
let admin_audit_events = purge_in_batches(
|
||||
&self.pool,
|
||||
"delete from admin_audit_events
|
||||
where created_at < $1 - ($2 * interval '1 day')",
|
||||
where ctid in (
|
||||
select ctid from admin_audit_events
|
||||
where created_at < $1 - ($2 * interval '1 day')
|
||||
limit $3
|
||||
)",
|
||||
now,
|
||||
settings.audit_retention_days,
|
||||
"delete expired admin audit events",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(i64::from(settings.audit_retention_days))
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("delete expired admin audit events"))?
|
||||
.rows_affected();
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("commit retention transaction"))?;
|
||||
.await?;
|
||||
Ok(RetentionReport {
|
||||
model_requests,
|
||||
ops_events,
|
||||
@@ -127,6 +129,32 @@ impl RetentionRepository for PgRetentionRepository {
|
||||
}
|
||||
}
|
||||
|
||||
const RETENTION_DELETE_BATCH_ROWS: i64 = 10_000;
|
||||
|
||||
async fn purge_in_batches(
|
||||
pool: &PgPool,
|
||||
delete_sql: &'static str,
|
||||
now: DateTime<Utc>,
|
||||
retention_days: u32,
|
||||
label: &'static str,
|
||||
) -> StoreResult<u64> {
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let deleted = sqlx::query(delete_sql)
|
||||
.bind(now)
|
||||
.bind(i64::from(retention_days))
|
||||
.bind(RETENTION_DELETE_BATCH_ROWS)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable(label))?
|
||||
.rows_affected();
|
||||
total += deleted;
|
||||
if deleted < RETENTION_DELETE_BATCH_ROWS.unsigned_abs() {
|
||||
return Ok(total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_u32(value: i64) -> StoreResult<u32> {
|
||||
u32::try_from(value).map_err(|_| StoreError::InvalidData {
|
||||
entity: "retention settings",
|
||||
|
||||
@@ -12,7 +12,7 @@ use gateway_core::provider_ports::{
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
};
|
||||
|
||||
use crate::{ConflictKind, Revision, StoreError, StoreResult, postgres_unavailable};
|
||||
use crate::{Revision, StoreError, StoreResult, postgres_unavailable};
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct RuntimeSettings {
|
||||
@@ -111,11 +111,8 @@ impl RuntimeSettingsUpdate {
|
||||
pub trait RuntimeSettingsRepository: Send + Sync {
|
||||
async fn load_runtime_settings(&self) -> StoreResult<RuntimeSettings>;
|
||||
|
||||
async fn update_runtime_settings(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
update: RuntimeSettingsUpdate,
|
||||
) -> StoreResult<Revision>;
|
||||
async fn update_runtime_settings(&self, update: RuntimeSettingsUpdate)
|
||||
-> StoreResult<Revision>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -169,7 +166,6 @@ impl RuntimeSettingsRepository for PgRuntimeSettingsRepository {
|
||||
|
||||
async fn update_runtime_settings(
|
||||
&self,
|
||||
expected_revision: Revision,
|
||||
update: RuntimeSettingsUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
let mut transaction = self
|
||||
@@ -177,9 +173,7 @@ impl RuntimeSettingsRepository for PgRuntimeSettingsRepository {
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("begin runtime settings update"))?;
|
||||
let revision =
|
||||
update_runtime_settings_in_transaction(&mut transaction, expected_revision, &update)
|
||||
.await?;
|
||||
let revision = update_runtime_settings_in_transaction(&mut transaction, &update).await?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
@@ -228,28 +222,26 @@ pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
|
||||
pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
expected_revision: Revision,
|
||||
update: &RuntimeSettingsUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
update.validate()?;
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update runtime_settings
|
||||
set config_revision = config_revision + 1,
|
||||
admin_api_key = $2,
|
||||
refresh_margin_seconds = $3,
|
||||
refresh_concurrency = $4,
|
||||
max_concurrent_per_account = $5,
|
||||
request_interval_ms = $6,
|
||||
rotation_strategy = $7,
|
||||
model_mappings_json = $8,
|
||||
usage_retention_days = $9,
|
||||
ops_event_retention_days = $10,
|
||||
audit_retention_days = $11,
|
||||
updated_at = now()
|
||||
where id = 1 and config_revision = $1
|
||||
returning config_revision",
|
||||
admin_api_key = $1,
|
||||
refresh_margin_seconds = $2,
|
||||
refresh_concurrency = $3,
|
||||
max_concurrent_per_account = $4,
|
||||
request_interval_ms = $5,
|
||||
rotation_strategy = $6,
|
||||
model_mappings_json = $7,
|
||||
usage_retention_days = $8,
|
||||
ops_event_retention_days = $9,
|
||||
audit_retention_days = $10,
|
||||
updated_at = now()
|
||||
where id = 1
|
||||
returning config_revision",
|
||||
)
|
||||
.bind(i64::try_from(expected_revision.get()).map_err(|_| invalid_numeric())?)
|
||||
.bind(update.admin_api_key.as_deref())
|
||||
.bind(i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?)
|
||||
.bind(i64::from(update.refresh_concurrency))
|
||||
@@ -263,32 +255,28 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("update runtime settings in transaction"))?
|
||||
.ok_or_else(|| StoreError::Conflict {
|
||||
.ok_or_else(|| StoreError::NotFound {
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
kind: ConflictKind::StaleRevision,
|
||||
})?;
|
||||
Revision::new(u64::try_from(next).map_err(|_| invalid_numeric())?)
|
||||
}
|
||||
|
||||
pub(crate) async fn bump_config_revision_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
expected_revision: Revision,
|
||||
) -> StoreResult<Revision> {
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update runtime_settings
|
||||
set config_revision = config_revision + 1, updated_at = now()
|
||||
where id = 1 and config_revision = $1
|
||||
where id = 1
|
||||
returning config_revision",
|
||||
)
|
||||
.bind(i64::try_from(expected_revision.get()).map_err(|_| invalid_numeric())?)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("bump config revision in transaction"))?
|
||||
.ok_or_else(|| StoreError::Conflict {
|
||||
.ok_or_else(|| StoreError::NotFound {
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
kind: ConflictKind::StaleRevision,
|
||||
})?;
|
||||
Revision::new(u64::try_from(next).map_err(|_| invalid_numeric())?)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! PostgreSQL 账号 cooldown 事实的可丢失 Redis 热缓存。
|
||||
//! 请求调度用的可丢失账号 cooldown Redis 存储。
|
||||
|
||||
use std::{sync::Arc, time::SystemTime};
|
||||
|
||||
@@ -243,7 +243,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
}
|
||||
}
|
||||
|
||||
/// PostgreSQL 账号事实到可丢失 cooldown cache 的 write-through adapter。
|
||||
/// 账号状态写入到可丢失 cooldown cache 的桥接 adapter。
|
||||
pub struct CooldownCachingProviderAccountStore {
|
||||
authoritative: Arc<dyn ProviderAccountStore>,
|
||||
cooldowns: Arc<dyn CredentialCooldownRepository>,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Provider account availability 的可重建 Redis cache fence。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
@@ -28,6 +30,12 @@ redis.call('PEXPIRE', KEYS[1], ARGV[5])
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const RECORD_REFRESH_BACKOFF_SCRIPT: &str = r#"
|
||||
local n = redis.call('INCR', KEYS[1])
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[1])
|
||||
return n
|
||||
"#;
|
||||
|
||||
const MAX_PROVIDER_CATALOG_BYTES: usize = 1024 * 1024;
|
||||
const CREDENTIAL_STATE_TTL_SECONDS: u64 = 24 * 60 * 60;
|
||||
|
||||
@@ -132,6 +140,14 @@ impl RedisCredentialStateRepository {
|
||||
Ok(format!("{}:account:{fingerprint}:state", self.namespace))
|
||||
}
|
||||
|
||||
fn backoff_key(&self, provider_account_id: &str) -> StoreResult<String> {
|
||||
let fingerprint = resource_fingerprint("credential refresh backoff", provider_account_id)?;
|
||||
Ok(format!(
|
||||
"{}:account:{fingerprint}:refresh-backoff",
|
||||
self.namespace
|
||||
))
|
||||
}
|
||||
|
||||
fn catalog_key(&self, key: &RedisProviderCatalogCacheKey) -> StoreResult<String> {
|
||||
key.validate()?;
|
||||
let fingerprint = resource_fingerprint("provider catalog cache", &key.catalog_scope)?;
|
||||
@@ -340,6 +356,48 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
|
||||
.map_err(|_| provider_unavailable("clear credential state"))
|
||||
})
|
||||
}
|
||||
|
||||
fn record_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
) -> futures::future::BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let window_millis = u64::try_from(window.as_millis())
|
||||
.ok()
|
||||
.filter(|millis| *millis > 0)
|
||||
.ok_or_else(|| provider_invalid("record refresh backoff"))?;
|
||||
let key = self
|
||||
.backoff_key(account_id.as_str())
|
||||
.map_err(|_| provider_unavailable("record refresh backoff"))?;
|
||||
let mut connection = self.connection.clone();
|
||||
let count = Script::new(RECORD_REFRESH_BACKOFF_SCRIPT)
|
||||
.key(key)
|
||||
.arg(window_millis)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("record refresh backoff"))?;
|
||||
Ok(u32::try_from(count).unwrap_or(u32::MAX))
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let key = self
|
||||
.backoff_key(account_id.as_str())
|
||||
.map_err(|_| provider_unavailable("clear refresh backoff"))?;
|
||||
let mut connection = self.connection.clone();
|
||||
redis::cmd("DEL")
|
||||
.arg(key)
|
||||
.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("clear refresh backoff"))?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCatalogCachePort for RedisCredentialStateRepository {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 可丢失、可从 PostgreSQL 或 Provider 重建的 Redis 协调状态。
|
||||
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
use redis::aio::ConnectionManager;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -33,17 +33,6 @@ pub use runtime_change::*;
|
||||
use crate::{StoreError, StoreResult, redis_unavailable, require_nonempty};
|
||||
|
||||
pub(crate) const MAX_REDIS_EXACT_INTEGER: u64 = (1_u64 << 53) - 1;
|
||||
const RECORD_ADMIN_LOGIN_FAILURE_SCRIPT: &str = r#"
|
||||
local count = redis.call('INCR', KEYS[1])
|
||||
local ttl = redis.call('PTTL', KEYS[1])
|
||||
if count == 1 or ttl < 0 then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
end
|
||||
if count >= tonumber(ARGV[1]) then
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
"#;
|
||||
|
||||
/// Redis 中可丢失的管理员会话事实;认证秘密不属于该结构。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -71,7 +60,7 @@ impl AdminSessionRecord {
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理员会话与登录失败窗口的 Redis 基础设施端口。
|
||||
/// 管理员会话的 Redis 基础设施端口。
|
||||
#[async_trait]
|
||||
pub trait AdminAuthStateRepository: Send + Sync {
|
||||
async fn load_admin_session(&self, session_id: &str)
|
||||
@@ -85,22 +74,9 @@ pub trait AdminAuthStateRepository: Send + Sync {
|
||||
&self,
|
||||
session_id: &str,
|
||||
) -> StoreResult<Option<AdminSessionRecord>>;
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> StoreResult<bool>;
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> StoreResult<bool>;
|
||||
async fn clear_login_failures(&self, source: &str) -> StoreResult<()>;
|
||||
}
|
||||
|
||||
/// Redis 管理员会话与登录失败窗口 adapter。
|
||||
/// Redis 管理员会话 adapter。
|
||||
#[derive(Clone)]
|
||||
pub struct RedisAdminAuthStateRepository {
|
||||
connection: ConnectionManager,
|
||||
@@ -119,11 +95,6 @@ impl RedisAdminAuthStateRepository {
|
||||
let fingerprint = resource_fingerprint("admin session", session_id)?;
|
||||
Ok(format!("{}:session:{{{fingerprint}}}", self.namespace))
|
||||
}
|
||||
|
||||
fn failure_key(&self, source: &str) -> StoreResult<String> {
|
||||
let fingerprint = resource_fingerprint("admin login failure", source)?;
|
||||
Ok(format!("{}:failure:{{{fingerprint}}}", self.namespace))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -179,54 +150,6 @@ impl AdminAuthStateRepository for RedisAdminAuthStateRepository {
|
||||
.map(|value| decode_admin_session(&value))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn login_source_is_throttled(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> StoreResult<bool> {
|
||||
validate_failure_policy(failure_limit, window_seconds)?;
|
||||
let key = self.failure_key(source)?;
|
||||
let mut connection = self.connection.clone();
|
||||
let failures = redis::cmd("GET")
|
||||
.arg(key)
|
||||
.query_async::<Option<u64>>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("load admin login failure window"))?
|
||||
.unwrap_or_default();
|
||||
Ok(failures >= u64::from(failure_limit))
|
||||
}
|
||||
|
||||
async fn record_login_failure(
|
||||
&self,
|
||||
source: &str,
|
||||
failure_limit: u32,
|
||||
window_seconds: u64,
|
||||
) -> StoreResult<bool> {
|
||||
let window_millis = validate_failure_policy(failure_limit, window_seconds)?;
|
||||
let key = self.failure_key(source)?;
|
||||
let mut connection = self.connection.clone();
|
||||
let throttled = Script::new(RECORD_ADMIN_LOGIN_FAILURE_SCRIPT)
|
||||
.key(key)
|
||||
.arg(failure_limit)
|
||||
.arg(window_millis)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("record admin login failure"))?;
|
||||
Ok(throttled == 1)
|
||||
}
|
||||
|
||||
async fn clear_login_failures(&self, source: &str) -> StoreResult<()> {
|
||||
let key = self.failure_key(source)?;
|
||||
let mut connection = self.connection.clone();
|
||||
redis::cmd("DEL")
|
||||
.arg(key)
|
||||
.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("clear admin login failures"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -259,17 +182,6 @@ fn decode_admin_session(value: &str) -> StoreResult<AdminSessionRecord> {
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_failure_policy(failure_limit: u32, window_seconds: u64) -> StoreResult<u64> {
|
||||
if failure_limit == 0 {
|
||||
return Err(admin_auth_invalid("login failure limit must be positive"));
|
||||
}
|
||||
let window_millis = window_seconds
|
||||
.checked_mul(1_000)
|
||||
.filter(|value| *value > 0 && *value <= MAX_REDIS_EXACT_INTEGER)
|
||||
.ok_or_else(|| admin_auth_invalid("login failure window is outside the supported range"))?;
|
||||
Ok(window_millis)
|
||||
}
|
||||
|
||||
fn admin_auth_invalid(message: &str) -> StoreError {
|
||||
StoreError::InvalidData {
|
||||
entity: "admin authentication state",
|
||||
|
||||
@@ -60,7 +60,7 @@ end
|
||||
return 1
|
||||
"#;
|
||||
|
||||
/// Redis-backed, best-effort native response continuation affinity.
|
||||
/// 基于 Redis 的 best-effort 原生响应 continuation 亲和。
|
||||
#[derive(Clone)]
|
||||
pub struct RedisNativeContinuationRepository {
|
||||
connection: ConnectionManager,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user