mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a32c6cec2 | ||
|
|
0ff1a72919 | ||
|
|
79083a5632 | ||
|
|
90f7456db8 | ||
|
|
6f71b27c3c | ||
|
|
7b05b14a2b | ||
|
|
f9b5b5ff77 | ||
|
|
4d5778a84f | ||
|
|
6d22d73506 | ||
|
|
d8ab207282 | ||
|
|
3816b46ba1 | ||
|
|
4a7c06f895 | ||
|
|
e0b7a841ca | ||
|
|
96a292dd0f | ||
|
|
6f138c8256 | ||
|
|
c8ffb1a8c8 | ||
|
|
79df82a4f0 | ||
|
|
62516d9769 | ||
|
|
892cba68ae | ||
|
|
fada9bc0bd | ||
|
|
27b9bbe994 | ||
|
|
db849c1251 | ||
|
|
c57af9b0dd |
@@ -7,21 +7,18 @@ RUN --mount=type=bind,target=/cache \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& cp -a /cache/registry/. /usr/local/cargo/registry/ \
|
||||
&& cp -a /cache/git/. /usr/local/cargo/git/ \
|
||||
&& cp -a /cache/target/. /app/backend/target/
|
||||
&& tar -xzf /cache/registry.tar.gz -C /usr/local/cargo/registry \
|
||||
&& tar -xzf /cache/git.tar.gz -C /usr/local/cargo/git \
|
||||
&& tar -xzf /cache/target.tar.gz -C /app/backend/target
|
||||
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS collect
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS snapshot
|
||||
ARG CACHE_STAMP
|
||||
# 一次导出三个 mount,避免逐目录创建镜像、载入 Docker 再复制文件。
|
||||
# 直接压缩 mount 内容,构建层和 runner 只保存归档,避免多份展开的编译产物占满磁盘
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& mkdir -p /cache/registry /cache/git /cache/target \
|
||||
&& cp -a /usr/local/cargo/registry/. /cache/registry/ \
|
||||
&& cp -a /usr/local/cargo/git/. /cache/git/ \
|
||||
&& cp -a /app/backend/target/. /cache/target/
|
||||
|
||||
FROM scratch AS snapshot
|
||||
COPY --from=collect /cache/ /
|
||||
&& mkdir -p /cache \
|
||||
&& tar -czf /cache/registry.tar.gz -C /usr/local/cargo/registry . \
|
||||
&& tar -czf /cache/git.tar.gz -C /usr/local/cargo/git . \
|
||||
&& tar -czf /cache/target.tar.gz -C /app/backend/target .
|
||||
|
||||
@@ -14,24 +14,42 @@ case "$architecture" in amd64|arm64) ;; *) exit 2 ;; esac
|
||||
dockerfile="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Dockerfile"
|
||||
|
||||
usable() {
|
||||
local root="$1" path crates=false libraries=false
|
||||
# 目录或 buildstamp 存在不代表编译发生过,必须同时有下载包与目标依赖。
|
||||
for path in "$root"/registry/cache/*/*.crate; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then crates=true; break; fi
|
||||
done
|
||||
for path in "$root"/target/"$architecture"/release/deps/*.rlib; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then libraries=true; break; fi
|
||||
done
|
||||
[ "$crates" = true ] && [ "$libraries" = true ]
|
||||
# 流式检查归档中的非空依赖文件,校验时不在 runner 展开编译产物
|
||||
python3 - "$1" "$architecture" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
architecture = sys.argv[2]
|
||||
|
||||
def contains(filename, pattern):
|
||||
with tarfile.open(root / filename, mode="r|gz") as archive:
|
||||
return any(
|
||||
member.isfile() and member.size > 0
|
||||
and re.fullmatch(pattern, member.name.removeprefix("./"))
|
||||
for member in archive
|
||||
)
|
||||
|
||||
try:
|
||||
valid = (
|
||||
(root / "git.tar.gz").stat().st_size > 0
|
||||
and contains("registry.tar.gz", r"cache/[^/]+/[^/]+\.crate")
|
||||
and contains("target.tar.gz", rf"{architecture}/release/deps/[^/]+\.rlib")
|
||||
)
|
||||
except (OSError, tarfile.TarError):
|
||||
valid = False
|
||||
sys.exit(0 if valid else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
command=(docker buildx build --builder "$builder" --file "$dockerfile" --progress plain)
|
||||
# mount 的内容不参与层指纹,用时间戳重跑复制;--no-cache 会重置 cache mount。
|
||||
# mount 的内容不参与层指纹,用时间戳重跑归档操作;--no-cache 会重置 cache mount
|
||||
command+=(--build-arg "CACHE_STAMP=$(date +%s%N)" --target "$operation")
|
||||
valid=false
|
||||
if [ "$operation" = restore ]; then
|
||||
if usable "$directory"; then
|
||||
mkdir -p "$directory"/{registry,git,target}
|
||||
"${command[@]}" --output type=cacheonly "$directory"
|
||||
valid=true
|
||||
else
|
||||
@@ -43,10 +61,11 @@ else
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,只接收 /cache 下的压缩归档
|
||||
# 直接导出打包阶段,避免再创建一层归档副本
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
| tar -xf - -C "$temporary/snapshot" --strip-components=1 cache/
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
@@ -36,10 +36,9 @@ jobs:
|
||||
with:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
# 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译
|
||||
key: container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
key: container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
restore-keys: |
|
||||
container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
- name: Restore Cargo mounts to BuildKit
|
||||
id: cargo-restore
|
||||
@@ -50,6 +49,9 @@ jobs:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
architecture: amd64
|
||||
|
||||
- name: Release restored Cargo snapshot
|
||||
run: rm -rf "$RUNNER_TEMP/container-cargo"
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail
|
||||
--test-threads 8 --no-tests fail
|
||||
|
||||
- name: Store tests with Redis ACL authentication
|
||||
working-directory: backend
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
--test-threads 8 --no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
|
||||
frontend:
|
||||
if: inputs.run_frontend
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
Generated
+40
-38
@@ -286,9 +286,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.155.0"
|
||||
version = "1.156.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a542f0e9261b9ce56708d591a62062079fa3c58a362ffbf49a2e48a4f6ced16"
|
||||
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.3"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -387,9 +387,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.64.2"
|
||||
version = "0.64.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398"
|
||||
checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -408,9 +408,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.5.1"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e"
|
||||
checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -432,9 +432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666"
|
||||
checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -452,9 +452,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -475,9 +475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.1"
|
||||
version = "1.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -492,9 +492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
|
||||
checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -514,9 +514,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -537,9 +537,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.64.0"
|
||||
version = "0.64.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413"
|
||||
checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -2485,9 +2485,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.0"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e7305938cac5ee144352ceb71583ffc6cfeac7d20fcb20ef865a4d5c314e499"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2513,18 +2513,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8a848331614144c03e815fb931f238c9e342bd7ab501b29e35d647e7cd5688"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5a947ef798eeacacb43edf29b1d0ab64bdf5e2c9fbcaad10ee8663d922b544"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -3578,9 +3578,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.1"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -3634,9 +3634,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "922268812b6f9e20b4ab0f1efe8e07036958eed7346ceddb31c777b5531fd11f"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3759,9 +3759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.21.1"
|
||||
version = "0.21.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
|
||||
checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -4953,9 +4953,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
@@ -5052,9 +5052,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.28.0"
|
||||
version = "1.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5646,16 +5646,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -49,6 +49,10 @@ opt-level = 2
|
||||
[profile.dev.package."miniz_oxide@0.9.1"]
|
||||
opt-level = 2
|
||||
|
||||
# workspace 的 WebSocket deflate 特性会让 flate2 选用 zlib-rs
|
||||
[profile.dev.package.zlib-rs]
|
||||
opt-level = 2
|
||||
|
||||
[profile.release]
|
||||
debug = false
|
||||
strip = "symbols"
|
||||
|
||||
@@ -70,6 +70,7 @@ fn gateway_core_depends_on_no_http_db_redis_or_provider_crate() {
|
||||
#[test]
|
||||
fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let allowed: Option<&[&str]> = match relative.to_str().expect("source path") {
|
||||
"validation.rs" => Some(&[]),
|
||||
@@ -96,7 +97,7 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let Some(allowed) = allowed else { continue };
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
let syntax = syn::parse_file(&source).expect("parse core source");
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
// 相对路径中的 self 指向当前 owner,不是跨 owner 依赖
|
||||
references.owners.remove(&references.module_path[0]);
|
||||
@@ -111,17 +112,24 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_owner_paths_form_a_dag() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
fn domain_owner_paths_form_a_dag() {
|
||||
for member in ["crates/gateway-core", "crates/gateway-admin"] {
|
||||
assert_owner_paths_form_a_dag(member);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_owner_paths_form_a_dag(member: &str) {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
let mut graph = BTreeMap::<String, BTreeSet<String>>::new();
|
||||
for relative in super::rust_files(&root) {
|
||||
if relative == Path::new("lib.rs") {
|
||||
continue;
|
||||
}
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
let owner = references.module_path[0].clone();
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse core source"));
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
references.owners.remove(&owner);
|
||||
graph.entry(owner).or_default().extend(references.owners);
|
||||
}
|
||||
@@ -131,7 +139,7 @@ fn core_owner_paths_form_a_dag() {
|
||||
while let Some(dependency) = pending.pop() {
|
||||
assert_ne!(
|
||||
&dependency, owner,
|
||||
"Core owner dependency cycle reaches {owner}"
|
||||
"{member} owner dependency cycle reaches {owner}"
|
||||
);
|
||||
if visited.insert(dependency.clone())
|
||||
&& let Some(next) = graph.get(&dependency)
|
||||
@@ -143,7 +151,44 @@ fn core_owner_paths_form_a_dag() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
fn shared_owners_do_not_depend_on_their_callers() {
|
||||
let boundaries: &[(&str, &str, &[&str])] = &[
|
||||
("crates/providers/xai", "transport", &["credential"]),
|
||||
("crates/providers/openai", "transport", &["provider"]),
|
||||
("crates/gateway-api", "time", &["admin"]),
|
||||
(
|
||||
"crates/gateway-api",
|
||||
"client_error",
|
||||
&["admin", "openai", "middleware"],
|
||||
),
|
||||
("crates/gateway-api", "middleware", &["openai"]),
|
||||
("crates/gateway-host", "system_update", &["config"]),
|
||||
("crates/gateway-plugin/runtime", "package", &["callback"]),
|
||||
];
|
||||
for &(member, owner, forbidden) in boundaries {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let mut references =
|
||||
CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
if references.module_path.first().map(String::as_str) != Some(owner) {
|
||||
continue;
|
||||
}
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
for dependency in forbidden {
|
||||
assert!(
|
||||
!references.owners.contains(*dependency),
|
||||
"{member}/src/{} must not depend on crate::{dependency}",
|
||||
relative.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
pub use crate::{engine::AttemptContext, settings::SettingsValues as Values};
|
||||
@@ -167,13 +212,86 @@ fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_crate_root_reexports() {
|
||||
let root = syn::parse_file(
|
||||
r#"
|
||||
mod provider;
|
||||
mod transport;
|
||||
mod secret;
|
||||
pub use provider::{OFFICIAL_URL, Options as ProviderOptions};
|
||||
pub use self::transport::{self as network, Request};
|
||||
pub use crate::secret::SecretValue;
|
||||
pub use external::Unrelated;
|
||||
"#,
|
||||
)
|
||||
.expect("parse root reexport fixture");
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
use crate::{OFFICIAL_URL, ProviderOptions, network::Client};
|
||||
use super::super::SecretValue;
|
||||
fn request() -> crate::Request { unimplemented!() }
|
||||
"#,
|
||||
)
|
||||
.expect("parse reexport consumer fixture");
|
||||
let reexports = root_reexport_owners(&root);
|
||||
assert!(!reexports.contains_key("Unrelated"));
|
||||
let mut references = CrateReferences::in_module(Path::new("consumer/request.rs"))
|
||||
.with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
assert_eq!(
|
||||
references.owners,
|
||||
BTreeSet::from([
|
||||
"provider".to_owned(),
|
||||
"transport".to_owned(),
|
||||
"secret".to_owned(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences {
|
||||
fn read_root_reexport_owners(root: &Path) -> BTreeMap<String, String> {
|
||||
let source = fs::read_to_string(root.join("lib.rs")).expect("read crate root");
|
||||
root_reexport_owners(&syn::parse_file(&source).expect("parse crate root"))
|
||||
}
|
||||
|
||||
// 只解析根模块的显式重导出,不推断 glob、宏展开或完整 Rust 名称解析
|
||||
fn root_reexport_owners(syntax: &syn::File) -> BTreeMap<String, String> {
|
||||
let modules = syntax
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
Item::Mod(module) => Some(module.ident.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut reexports = BTreeMap::new();
|
||||
for item in &syntax.items {
|
||||
if let Item::Use(item) = item
|
||||
&& !matches!(item.vis, syn::Visibility::Inherited)
|
||||
{
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, name| {
|
||||
let owner = path
|
||||
.iter()
|
||||
.find(|segment| !matches!(segment.as_str(), "crate" | "self"));
|
||||
if let (Some(owner), Some(name)) = (owner, name)
|
||||
&& modules.contains(owner)
|
||||
{
|
||||
reexports.insert(name.to_owned(), owner.clone());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
reexports
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences<'a> {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
root_reexports: Option<&'a BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences<'_> {
|
||||
fn visit_path(&mut self, path: &'ast syn::Path) {
|
||||
self.record(
|
||||
&path
|
||||
@@ -186,11 +304,13 @@ impl<'ast> Visit<'ast> for CrateReferences {
|
||||
}
|
||||
|
||||
fn visit_item_use(&mut self, item: &'ast syn::ItemUse) {
|
||||
self.use_tree(&item.tree, &mut Vec::new());
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, _| {
|
||||
self.record(path);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl CrateReferences {
|
||||
impl<'a> CrateReferences<'a> {
|
||||
fn in_module(relative: &Path) -> Self {
|
||||
let mut module = relative
|
||||
.with_extension("")
|
||||
@@ -208,13 +328,26 @@ impl CrateReferences {
|
||||
Self {
|
||||
owners: BTreeSet::new(),
|
||||
module_path: module,
|
||||
root_reexports: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_root_reexports(mut self, reexports: &'a BTreeMap<String, String>) -> Self {
|
||||
self.root_reexports = Some(reexports);
|
||||
self
|
||||
}
|
||||
|
||||
fn root_owner(&self, name: &str) -> String {
|
||||
self.root_reexports
|
||||
.and_then(|reexports| reexports.get(name))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| name.to_owned())
|
||||
}
|
||||
|
||||
fn record(&mut self, path: &[String]) {
|
||||
let Some(first) = path.first() else { return };
|
||||
let owner = if first == "crate" {
|
||||
path.get(1).cloned()
|
||||
path.get(1).map(|name| self.root_owner(name))
|
||||
} else if first == "self" || first == "super" {
|
||||
let mut module = self.module_path.clone();
|
||||
let mut rest = path;
|
||||
@@ -227,7 +360,10 @@ impl CrateReferences {
|
||||
}
|
||||
rest = &rest[1..];
|
||||
}
|
||||
module.first().or_else(|| rest.first()).cloned()
|
||||
module
|
||||
.first()
|
||||
.cloned()
|
||||
.or_else(|| rest.first().map(|name| self.root_owner(name)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -235,31 +371,41 @@ impl CrateReferences {
|
||||
self.owners.insert(owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn use_tree(&mut self, tree: &syn::UseTree, prefix: &mut Vec<String>) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
self.use_tree(&path.tree, prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
prefix.push(name.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
self.use_tree(item, prefix);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => self.record(prefix),
|
||||
fn visit_use_leaves(
|
||||
tree: &syn::UseTree,
|
||||
prefix: &mut Vec<String>,
|
||||
visit: &mut impl FnMut(&[String], Option<&str>),
|
||||
) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
visit_use_leaves(&path.tree, prefix, visit);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
let name = name.ident.to_string();
|
||||
let export = if name == "self" {
|
||||
prefix.last().cloned()
|
||||
} else {
|
||||
Some(name.clone())
|
||||
};
|
||||
prefix.push(name);
|
||||
visit(prefix, export.as_deref());
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
visit(prefix, Some(&rename.rename.to_string()));
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
visit_use_leaves(item, prefix, visit);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => visit(prefix, None),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,11 +11,15 @@ use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerDaemonContext, WorkerTaskError},
|
||||
task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId,
|
||||
WorkerKind, WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStatusTransition,
|
||||
BackupStorageConfig, BackupTriggerKind, build_backup_seed, code,
|
||||
@@ -34,6 +38,8 @@ impl From<BackupError> for WorkerTaskError {
|
||||
|
||||
use super::policy::{BackupSchedule, decide_retention};
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
|
||||
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量
|
||||
@@ -72,6 +78,22 @@ impl BackupTask {
|
||||
self.timezone = timezone;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
//! 管理服务与后台任务的启动组装,窄端口供服务器和 CLI 复用
|
||||
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{engine::probe::AccountProbe, runtime::SnapshotControl};
|
||||
|
||||
use crate::{
|
||||
AccountsService, AdminBundle, AdminConfig, AdminRuntimePorts, AdminServices, ClientConfig,
|
||||
ClientKeyService, CredentialsService, PluginDistributionPorts, PluginManagementService,
|
||||
PluginsService, SettingsService, UsageDataService, backup, freeze_recovery,
|
||||
model::AdminError,
|
||||
ports::{
|
||||
self, plugin_accounts::PluginAccountAccess, plugin_client_keys::PluginClientKeyAccess,
|
||||
provider::ProviderAdminRegistry, store::AdminStorePorts,
|
||||
},
|
||||
public_service, reset_credit_worker, use_case,
|
||||
use_case::{
|
||||
account_groups::DefaultAccountGroupService,
|
||||
accounts::DefaultAccountsService,
|
||||
auth::{AuthService, DefaultAuthService},
|
||||
backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService,
|
||||
client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService,
|
||||
settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
},
|
||||
};
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
providers.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(
|
||||
use_case::system_preflight::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
),
|
||||
);
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), providers.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
providers.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
providers.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let worker_contributions = vec![
|
||||
backup_task.into_worker()?,
|
||||
import_task.into_worker()?,
|
||||
freeze_recovery.into_worker()?,
|
||||
reset_credit_worker::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}
|
||||
.into_worker()?,
|
||||
];
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
@@ -8,9 +8,13 @@ use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountErrorProtection, ProviderAccountId};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::AdminError;
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
@@ -42,6 +46,34 @@ impl FreezeRecoveryTask {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
FREEZE_RECOVERY_INTERVAL,
|
||||
WORKER_INITIAL_BACKOFF,
|
||||
WORKER_MAXIMUM_BACKOFF,
|
||||
WORKER_LEASE_TTL,
|
||||
WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
|
||||
/// 无法读取或校验配置时跳过本轮,不能把未知状态当作允许解冻
|
||||
async fn freeze_policy(&self) -> Option<AccountErrorProtection> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
|
||||
@@ -2,26 +2,22 @@
|
||||
//!
|
||||
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现
|
||||
|
||||
use std::{fmt, path::Path, sync::Arc, time::Duration};
|
||||
use std::{fmt, path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{
|
||||
engine::execution::ClientKeyVerifier,
|
||||
engine::probe::AccountProbe,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
engine::execution::ClientKeyVerifier, engine::probe::AccountProbe, runtime::SnapshotControl,
|
||||
task::WorkerContribution,
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
mod bootstrap;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
pub mod public_service;
|
||||
mod reset_credit_automation;
|
||||
mod reset_credit_worker;
|
||||
mod use_case;
|
||||
pub use use_case::plugins::{PluginDistributionPorts, PluginManagementService, PluginsService};
|
||||
|
||||
@@ -44,18 +40,15 @@ pub use use_case::{
|
||||
usage_data::UsageDataService,
|
||||
};
|
||||
|
||||
use model::AdminError;
|
||||
pub use bootstrap::{
|
||||
initialize, initialize_accounts, initialize_plugin_accounts, initialize_plugin_client_keys,
|
||||
initialize_plugin_resources, initialize_settings, initialize_usage_data,
|
||||
initialize_with_plugin_accounts,
|
||||
};
|
||||
|
||||
use ports::{
|
||||
client_distribution::ClientDistributionResolver, plugin_accounts::PluginAccountAccess,
|
||||
plugin_client_keys::PluginClientKeyAccess, provider::ProviderAdminRegistry,
|
||||
store::AdminStorePorts, system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
account_groups::DefaultAccountGroupService, accounts::DefaultAccountsService,
|
||||
auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService, client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService, settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
provider::ProviderAdminRegistry, system::SystemOperations,
|
||||
};
|
||||
|
||||
const MINIMUM_INITIAL_PASSWORD_BYTES: usize = 12;
|
||||
@@ -70,7 +63,6 @@ const WEAK_ADMIN_PASSWORDS: &[&str] = &[
|
||||
"codex-proxy-rs",
|
||||
];
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
const DEFAULT_CLIENT_SESSION_TTL_MINUTES: u64 = 24 * 60;
|
||||
|
||||
/// 只用于首次幂等创建默认管理员的启动密码
|
||||
@@ -366,395 +358,3 @@ pub struct AdminRuntimePorts {
|
||||
pub system: Arc<dyn SystemOperations>,
|
||||
pub client_key_verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
let registry = providers;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(use_case::plugin_update::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
));
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), registry.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
registry.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(import_task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(reset_credit_automation::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?,
|
||||
));
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`
|
||||
fn backup_worker_contribution(
|
||||
task: backup::task::BackupTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
//! 每日错峰发现与已知卡片的临期执行,周期只读取本地到期计划
|
||||
|
||||
use crate::{
|
||||
model::reset_credit_automation::{PlannedResetCredit, ResetCreditAutomation},
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{ScheduledTask, WorkerCycleContext, WorkerTaskError},
|
||||
time::DeploymentTimeZone,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
pub(crate) fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
pub(crate) fn consume_at(
|
||||
account_id: &str,
|
||||
credit: &PlannedResetCredit,
|
||||
lead_minutes: u32,
|
||||
) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
pub(crate) fn next_consumption(
|
||||
record: &ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
pub(crate) fn reschedule(
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//! 自动重置卡 Worker 的到期扫描、并发预算与取消,消费编排由账号用例负责
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ResetCreditAutomationTask {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,13 @@
|
||||
//! 重置卡自动使用的账号锁、持久幂等键、查询缓存与重试编排
|
||||
//! 重置卡查询、消费与自动使用排程,统一账号锁、持久幂等键和重试编排
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::model::{MutationActor, reset_credit_automation::*};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::time::DeploymentTimeZone;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
pub(super) async fn reset_credits_upstream(
|
||||
&self,
|
||||
@@ -51,13 +57,6 @@ impl DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
use crate::{
|
||||
model::{MutationActor, reset_credit_automation::*},
|
||||
reset_credit_automation::{consume_at, next_consumption, next_discovery, reschedule},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
async fn automation_settings(
|
||||
&self,
|
||||
@@ -511,3 +510,54 @@ impl DefaultAccountsService {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
fn consume_at(account_id: &str, credit: &PlannedResetCredit, lead_minutes: u32) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
fn next_consumption(record: &ResetCreditAutomation, lead_minutes: u32) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
fn reschedule(record: &mut ResetCreditAutomation, lead_minutes: u32, now: DateTime<Utc>) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -9,7 +9,10 @@ use std::{
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::task::{DaemonTask, WorkerDaemonContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId, WorkerKind,
|
||||
WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -328,6 +331,24 @@ impl ImportTasksService for DefaultImportTasksService {
|
||||
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl ImportTaskWorker {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
|
||||
@@ -14,11 +14,11 @@ pub mod peer_link;
|
||||
pub mod plugin_accounts;
|
||||
pub mod plugin_client_keys;
|
||||
pub mod plugin_resources;
|
||||
pub(crate) mod plugin_update;
|
||||
pub mod plugins;
|
||||
pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub(crate) mod system_preflight;
|
||||
pub mod usage_data;
|
||||
|
||||
mod credential_mutation;
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
peer_link::{PeerLinkExport, PeerLinkLimits},
|
||||
provider_credentials::ProviderQuotaReadRequest,
|
||||
provider_credentials::{ProviderQuotaReadRequest, explicit_plan_type},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
@@ -175,10 +175,16 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "peer account runtime"))?;
|
||||
let mut quotas = BTreeMap::new();
|
||||
if sharing.share_quota {
|
||||
if sharing.share_quota || sharing.share_plans {
|
||||
let mut window_count = 0usize;
|
||||
let mut by_provider = BTreeMap::new();
|
||||
for account in &accounts {
|
||||
// 仅公开套餐时只补读缺失套餐的本地额度,不回源刷新或扩大公开范围
|
||||
if !sharing.share_quota
|
||||
&& explicit_plan_type(account.plan_type.as_deref()).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
by_provider
|
||||
.entry(&account.provider_kind)
|
||||
.or_insert_with(Vec::new)
|
||||
@@ -200,9 +206,11 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.zip(provider.read_quotas(&requests).await)
|
||||
{
|
||||
if let Ok(quota) = quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
if sharing.share_quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
}
|
||||
}
|
||||
quotas.insert(account.id.clone(), quota);
|
||||
}
|
||||
@@ -215,6 +223,7 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
if sharing.share_pool {
|
||||
result.pool = Some(projection::pool_summary(
|
||||
&accounts,
|
||||
"as,
|
||||
&runtime,
|
||||
&self.providers,
|
||||
sharing.share_plans,
|
||||
|
||||
@@ -25,6 +25,7 @@ use crate::{
|
||||
|
||||
pub(super) fn pool_summary(
|
||||
accounts: &[AccountRecord],
|
||||
quotas: &BTreeMap<String, ProviderQuota>,
|
||||
runtime: &AccountRuntimeSnapshot,
|
||||
providers: &ProviderAdminRegistry,
|
||||
share_plans: bool,
|
||||
@@ -43,12 +44,13 @@ pub(super) fn pool_summary(
|
||||
AccountStatus::Error => summary.error += 1,
|
||||
}
|
||||
if let Some(plans) = &mut summary.plans
|
||||
&& let (Some(plan), Ok(provider)) = (
|
||||
&account.plan_type,
|
||||
providers.require(&account.provider_kind),
|
||||
&& let Some(plan) = providers.resolve_account_plan(
|
||||
account.provider_kind.as_str(),
|
||||
&mut account.plan_type.clone(),
|
||||
quotas.get(&account.id),
|
||||
)
|
||||
{
|
||||
*plans.entry(provider.plan_type_display(plan)).or_default() += 1;
|
||||
*plans.entry(plan).or_default() += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
|
||||
@@ -5,7 +5,7 @@ use super::{
|
||||
};
|
||||
use crate::model::provider_credentials::ProviderQuota;
|
||||
use gateway_core::{
|
||||
account::{QuotaObservation, UpstreamConnection},
|
||||
account::{QuotaObservation, UpstreamConnection, UpstreamInfoFailure},
|
||||
error::StoreError,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
@@ -56,6 +56,7 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling: None,
|
||||
summary: None,
|
||||
unsupported: false,
|
||||
},
|
||||
));
|
||||
}
|
||||
@@ -85,23 +86,25 @@ impl PeerInfoService {
|
||||
connections
|
||||
.iter()
|
||||
.map(|connection| {
|
||||
let cached = observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection);
|
||||
(
|
||||
connection.clone(),
|
||||
observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection)
|
||||
.and_then(|cached| cached.summary.clone()),
|
||||
cached.and_then(|cached| cached.summary.clone()),
|
||||
cached.is_some_and(|cached| cached.unsupported),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
Ok(summaries
|
||||
.into_iter()
|
||||
.map(|(connection, peer)| {
|
||||
.map(|(connection, peer, unsupported)| {
|
||||
let quota = Self::project_peer(
|
||||
&connection,
|
||||
peer.as_deref(),
|
||||
self.last_error(&connection),
|
||||
self.last_error(&connection)
|
||||
.or(unsupported.then_some(UpstreamInfoFailure::Unsupported)),
|
||||
now,
|
||||
);
|
||||
(connection, quota)
|
||||
@@ -132,6 +135,8 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling,
|
||||
summary,
|
||||
unsupported: observation
|
||||
.is_some_and(|value| snapshot::is_unsupported(&value.quota, connection)),
|
||||
};
|
||||
let previous = self
|
||||
.observations
|
||||
@@ -140,4 +145,12 @@ impl PeerInfoService {
|
||||
.insert(connection.account_id.clone(), cached);
|
||||
drop(previous);
|
||||
}
|
||||
|
||||
pub(super) fn unsupported(&self, connection: &UpstreamConnection) -> bool {
|
||||
self.observations
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&connection.account_id)
|
||||
.is_some_and(|cached| cached.connection == *connection && cached.unsupported)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,7 @@ use crate::model::{
|
||||
},
|
||||
};
|
||||
use gateway_core::account::{QuotaAccessState, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
PeerComponentStatus, PeerLink, PeerPoolDisclosure, PeerQuotaDisclosure,
|
||||
};
|
||||
use gateway_protocol::peer::{PeerComponentStatus, PeerLink};
|
||||
|
||||
use super::{PeerInfoService, quota, snapshot};
|
||||
|
||||
@@ -39,11 +37,11 @@ impl PeerInfoService {
|
||||
last_error: last_error.map(|error| error.as_str().to_owned()),
|
||||
pool: peer
|
||||
.as_ref()
|
||||
.filter(|peer| peer.pool.disclosure == PeerPoolDisclosure::Summary)
|
||||
.and_then(|peer| {
|
||||
let states = peer.pool.states.as_ref()?;
|
||||
Some(UpstreamPool {
|
||||
account_count: peer.pool.account_count?,
|
||||
.and_then(|peer| peer.pool.as_ref())
|
||||
.map(|pool| {
|
||||
let states = &pool.states;
|
||||
UpstreamPool {
|
||||
account_count: pool.account_count,
|
||||
states: UpstreamPoolStates {
|
||||
ready: states.ready,
|
||||
cooldown: states.cooldown,
|
||||
@@ -52,8 +50,7 @@ impl PeerInfoService {
|
||||
error: states.error,
|
||||
unknown: states.unknown,
|
||||
},
|
||||
plans: peer
|
||||
.pool
|
||||
plans: pool
|
||||
.plans
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
@@ -63,19 +60,18 @@ impl PeerInfoService {
|
||||
count: plan.count,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}),
|
||||
};
|
||||
let mut windows = Vec::new();
|
||||
if let Some(peer) = &peer {
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(unit) = &usage.unit
|
||||
{
|
||||
for (key, label, seconds, window) in [
|
||||
("daily", "日额度", 86400, &usage.daily),
|
||||
("weekly", "周额度", 604800, &usage.weekly),
|
||||
] {
|
||||
// 不限额仍有日、周账本,只是不计算占用百分比
|
||||
for window in &usage.windows {
|
||||
// 不限额仍展示已提供的账本窗口,只是不计算占用百分比
|
||||
let total = window
|
||||
.total
|
||||
.as_deref()
|
||||
@@ -84,19 +80,17 @@ impl PeerInfoService {
|
||||
let used_percent = snapshot::decimal(&window.used)
|
||||
.zip(total)
|
||||
.map(|(used, total)| (used / total * 100.0).clamp(0.0, 100.0));
|
||||
let window_observed = peer
|
||||
.usage
|
||||
let window_observed = component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
|| !peer
|
||||
.usage
|
||||
|| !component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.is_some_and(|value| snapshot::fresh(value, now))
|
||||
|| window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::timestamp)
|
||||
.is_some_and(|reset| reset <= now);
|
||||
@@ -106,27 +100,28 @@ impl PeerInfoService {
|
||||
observed_at: window_observed,
|
||||
stale: window_stale,
|
||||
amount: Some(UpstreamQuotaAmount {
|
||||
unit: usage.unit.clone(),
|
||||
unit: unit.clone(),
|
||||
used: window.used.clone(),
|
||||
total: window.total.clone(),
|
||||
remaining: window.remaining.clone(),
|
||||
}),
|
||||
}),
|
||||
key: format!("peer:budget:{key}"),
|
||||
key: format!("peer:budget:{}", window.key),
|
||||
group: "general".to_owned(),
|
||||
label: label.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: Some(if key == "weekly" {
|
||||
ProviderQuotaWindowRole::Secondary
|
||||
} else {
|
||||
ProviderQuotaWindowRole::Primary
|
||||
}),
|
||||
role: match window.key.as_str() {
|
||||
"daily" => Some(ProviderQuotaWindowRole::Primary),
|
||||
"weekly" => Some(ProviderQuotaWindowRole::Secondary),
|
||||
"monthly" => Some(ProviderQuotaWindowRole::Monthly),
|
||||
_ => None,
|
||||
},
|
||||
// Key 消费窗口覆盖此连接的全部请求,本机用量按同一时间边界聚合
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(seconds),
|
||||
window_seconds: window.window_seconds,
|
||||
used_percent,
|
||||
reset_at: window.resets_at.as_deref().and_then(snapshot::datetime),
|
||||
reset_at: window.reset_at.as_deref().and_then(snapshot::datetime),
|
||||
limit_reached: !window_stale
|
||||
&& quota::window_access(window, now) == QuotaAccessState::Exhausted,
|
||||
local_usage: None,
|
||||
@@ -135,8 +130,8 @@ impl PeerInfoService {
|
||||
}
|
||||
}
|
||||
// 池内窗口各自保留账号归属,不能将不同套餐相加为统一消费额度
|
||||
if peer.quota.disclosure == PeerQuotaDisclosure::Accounts {
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
if let Some(quota) = &peer.quota {
|
||||
for (index, account) in quota.accounts.iter().enumerate() {
|
||||
let window_observed =
|
||||
account.observed_at.as_deref().and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
@@ -146,6 +141,7 @@ impl PeerInfoService {
|
||||
now,
|
||||
);
|
||||
for window in &account.windows {
|
||||
let group = window.group.as_deref().unwrap_or("general");
|
||||
let window_stale = window_stale
|
||||
|| window
|
||||
.reset_at
|
||||
@@ -156,7 +152,7 @@ impl PeerInfoService {
|
||||
"peer:{}:{}:{}",
|
||||
account.id.len(),
|
||||
account.id,
|
||||
window.limit_id.as_deref().unwrap_or(&window.group)
|
||||
window.limit_id.as_deref().unwrap_or(group)
|
||||
);
|
||||
windows.push(ProviderQuotaWindow {
|
||||
upstream: Some(UpstreamQuotaWindow {
|
||||
@@ -166,8 +162,8 @@ impl PeerInfoService {
|
||||
amount: None,
|
||||
}),
|
||||
key: format!("{scope}:{}", window.key),
|
||||
group: window.group.clone(),
|
||||
label: window.label.clone(),
|
||||
group: group.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: Some(scope),
|
||||
limit_name: Some(match window.limit_name.as_deref() {
|
||||
Some(name) => format!("账号 {} · {name}", index + 1),
|
||||
|
||||
@@ -13,24 +13,26 @@ pub(super) fn observe(
|
||||
started_at: SystemTime,
|
||||
now: SystemTime,
|
||||
) -> Option<QuotaState> {
|
||||
let usage = peer.usage.as_ref()?;
|
||||
if !fresh(&peer.observed_at, now)
|
||||
|| !component_fresh(Some(&peer.observed_at), Some(&peer.expires_at), now)
|
||||
|| peer.usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(peer.usage.observed_at.as_deref(), None, now)
|
||||
|| usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(usage.observed_at.as_deref(), None, now)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let observed_at = started_at
|
||||
.min(timestamp(&peer.observed_at)?)
|
||||
.min(timestamp(peer.usage.observed_at.as_deref()?)?);
|
||||
let usage = peer.usage.data.as_ref()?;
|
||||
.min(timestamp(usage.observed_at.as_deref()?)?);
|
||||
let usage = usage.data.as_ref()?;
|
||||
match access(usage, now) {
|
||||
QuotaAccessState::Allowed => Some(QuotaState::allowed(observed_at)),
|
||||
QuotaAccessState::Exhausted => {
|
||||
let reset_at = [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
let reset_at = usage
|
||||
.windows
|
||||
.iter()
|
||||
.filter(|window| window_access(window, now) == QuotaAccessState::Exhausted)
|
||||
.filter_map(|window| window.resets_at.as_deref().and_then(timestamp))
|
||||
.filter_map(|window| window.reset_at.as_deref().and_then(timestamp))
|
||||
.min();
|
||||
Some(QuotaState::exhausted(
|
||||
if usage.allowed == Some(false) {
|
||||
@@ -51,8 +53,9 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
match usage.allowed {
|
||||
Some(true) => QuotaAccessState::Allowed,
|
||||
Some(false) => QuotaAccessState::Exhausted,
|
||||
None if [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
None if usage
|
||||
.windows
|
||||
.iter()
|
||||
.any(|window| window_access(window, now) == QuotaAccessState::Exhausted) =>
|
||||
{
|
||||
QuotaAccessState::Exhausted
|
||||
@@ -64,7 +67,7 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
/// 过期窗口保持未知,不从重置时间或金额推断恢复
|
||||
pub(super) fn window_access(window: &PeerBudgetWindow, now: SystemTime) -> QuotaAccessState {
|
||||
if window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(timestamp)
|
||||
.is_some_and(|reset| reset <= now)
|
||||
|
||||
@@ -43,6 +43,9 @@ impl PeerInfoService {
|
||||
}
|
||||
|
||||
pub(super) fn refresh_due(&self, connection: &UpstreamConnection) -> bool {
|
||||
if self.unsupported(connection) {
|
||||
return false;
|
||||
}
|
||||
let state = self.refresh_state(connection);
|
||||
state.try_lock().is_ok_and(|state| {
|
||||
state.connection != *connection || Instant::now() >= state.next_attempt
|
||||
@@ -91,17 +94,18 @@ impl PeerInfoService {
|
||||
.ok_or_else(|| ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))?;
|
||||
let slot = self.refresh_state(&connection);
|
||||
let mut state = slot.lock().await;
|
||||
if state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
if !manual && self.unsupported(&connection)
|
||||
|| state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
{
|
||||
let error = state.last_error;
|
||||
drop(state);
|
||||
if let Some(error) = error {
|
||||
if let Some(error) = error.filter(|error| *error != UpstreamInfoFailure::Unsupported) {
|
||||
return Err(refresh_error(error));
|
||||
}
|
||||
return self
|
||||
@@ -126,19 +130,18 @@ impl PeerInfoService {
|
||||
let failure = result.as_ref().err().copied();
|
||||
let delay = match failure {
|
||||
None => Duration::from_secs(240 + stable_jitter(id.as_str())),
|
||||
Some(UpstreamInfoFailure::Unauthorized | UpstreamInfoFailure::Unsupported) => {
|
||||
Duration::from_secs(1800)
|
||||
}
|
||||
Some(UpstreamInfoFailure::Unauthorized) => Duration::from_secs(1800),
|
||||
Some(_) => Duration::from_secs(30),
|
||||
}
|
||||
.max(retry_after.unwrap_or_default());
|
||||
state.next_attempt = Instant::now() + delay;
|
||||
state.manual_after = Instant::now()
|
||||
+ if failure.is_some() {
|
||||
+ (if failure.is_some_and(|error| error != UpstreamInfoFailure::Unsupported) {
|
||||
delay
|
||||
} else {
|
||||
Duration::from_secs(15)
|
||||
};
|
||||
})
|
||||
.max(retry_after.unwrap_or_default());
|
||||
state.last_error = failure;
|
||||
let document = match result {
|
||||
Ok(document) => Some(document),
|
||||
@@ -146,6 +149,7 @@ impl PeerInfoService {
|
||||
Err(UpstreamInfoFailure::Unauthorized) => {
|
||||
Some(OpaqueProviderData::new(Default::default()))
|
||||
}
|
||||
Err(UpstreamInfoFailure::Unsupported) => Some(snapshot::unsupported(&connection)),
|
||||
Err(_) => None,
|
||||
};
|
||||
if let Some(document) = document {
|
||||
@@ -185,7 +189,8 @@ impl PeerInfoService {
|
||||
);
|
||||
}
|
||||
drop(state);
|
||||
if let Some(error) = failure {
|
||||
// 不支持协议是正常发现结果,管理端保留分类,后台不重复报警
|
||||
if let Some(error) = failure.filter(|error| *error != UpstreamInfoFailure::Unsupported) {
|
||||
return Err(refresh_error(error));
|
||||
}
|
||||
self.read_quota(id)
|
||||
|
||||
@@ -70,22 +70,23 @@ pub(super) struct SchedulingObservation {
|
||||
impl SchedulingObservation {
|
||||
pub(super) fn new(peer: &PeerLink, now: SystemTime) -> Option<Self> {
|
||||
let validity = Validity::component(Some(&peer.observed_at), Some(&peer.expires_at))?;
|
||||
let mut exhausted_budgets = Vec::with_capacity(2);
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
let mut exhausted_budgets = Vec::new();
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(usage_validity) =
|
||||
Validity::component(peer.usage.observed_at.as_deref(), None)
|
||||
Validity::component(component.observed_at.as_deref(), None)
|
||||
&& quota::access(usage, now) == QuotaAccessState::Exhausted
|
||||
{
|
||||
if usage.allowed.is_some() {
|
||||
exhausted_budgets.push(usage_validity);
|
||||
} else {
|
||||
// 窗口提供的补充证据只在各自重置边界之前影响调度评分
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
for window in &usage.windows {
|
||||
if quota::window_access(window, now) == QuotaAccessState::Exhausted {
|
||||
let mut validity = usage_validity;
|
||||
if let Some(reset) =
|
||||
window.resets_at.as_deref().and_then(snapshot::timestamp)
|
||||
window.reset_at.as_deref().and_then(snapshot::timestamp)
|
||||
{
|
||||
validity.expires_at = validity.expires_at.min(reset);
|
||||
}
|
||||
@@ -97,7 +98,11 @@ impl SchedulingObservation {
|
||||
|
||||
// 同一账号重复列出模型不增加候选数,多账号覆盖不计算池平均百分比
|
||||
let mut model_accounts = HashMap::<&str, Option<usize>>::new();
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
let quota_accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
for (index, account) in quota_accounts.iter().enumerate() {
|
||||
for model in &account.models {
|
||||
model_accounts
|
||||
.entry(model)
|
||||
@@ -128,7 +133,7 @@ impl SchedulingObservation {
|
||||
accounts
|
||||
.entry(index)
|
||||
.or_insert_with(|| {
|
||||
let account = &peer.quota.accounts[index];
|
||||
let account = "a_accounts[index];
|
||||
let validity = Validity::component(
|
||||
account.observed_at.as_deref(),
|
||||
account.expires_at.as_deref(),
|
||||
|
||||
@@ -12,7 +12,7 @@ use futures::{StreamExt as _, stream};
|
||||
use gateway_core::{
|
||||
account::{
|
||||
LoadedCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
UpstreamConnection, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
UpstreamConnection, UpstreamInfoFailure, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
},
|
||||
error::{StoreError, StoreErrorKind},
|
||||
routing::ProviderKind,
|
||||
@@ -34,6 +34,7 @@ pub(super) struct CachedObservation {
|
||||
pub checked_at: SystemTime,
|
||||
pub scheduling: Option<Arc<SchedulingObservation>>,
|
||||
pub summary: Option<Arc<PeerLink>>,
|
||||
pub unsupported: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct PeerInfoService {
|
||||
@@ -171,13 +172,17 @@ impl PeerInfoService {
|
||||
let peer = observation
|
||||
.as_ref()
|
||||
.and_then(|value| snapshot::decode(&value.quota));
|
||||
let unsupported = observation
|
||||
.as_ref()
|
||||
.is_some_and(|value| snapshot::is_unsupported(&value.quota, &connection));
|
||||
if self.current_connection(id).await?.as_ref() != Some(&connection) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Self::project_peer(
|
||||
&connection,
|
||||
peer.as_ref(),
|
||||
self.last_error(&connection),
|
||||
self.last_error(&connection)
|
||||
.or(unsupported.then_some(UpstreamInfoFailure::Unsupported)),
|
||||
SystemTime::now(),
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -1,18 +1,46 @@
|
||||
//! 互联响应的有界校验、能力授权与时间有效性解释
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamInfoFailure};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerLink, PeerPoolDisclosure,
|
||||
SCHEMA_VERSION,
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerKeyUsage, PeerLink, PeerPool,
|
||||
PeerQuota, SCHEMA_VERSION,
|
||||
};
|
||||
use serde::Deserialize as _;
|
||||
|
||||
pub(super) const MAX_AGE: Duration = Duration::from_secs(30 * 60);
|
||||
pub(super) const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
// 不支持协议是连接的识别结果,复用观测存储与版本保护,不伪造上游额度快照
|
||||
pub(super) fn unsupported(connection: &UpstreamConnection) -> OpaqueProviderData {
|
||||
OpaqueProviderData::new(serde_json::Map::from_iter([
|
||||
("object".into(), "cpr.peer.unsupported".into()),
|
||||
("schema_version".into(), 1.into()),
|
||||
(
|
||||
"connection_revision".into(),
|
||||
connection.revision.clone().into(),
|
||||
),
|
||||
]))
|
||||
}
|
||||
|
||||
pub(super) fn is_unsupported(data: &OpaqueProviderData, connection: &UpstreamConnection) -> bool {
|
||||
let fields = data.expose_to_provider();
|
||||
fields.get("object").and_then(serde_json::Value::as_str) == Some("cpr.peer.unsupported")
|
||||
&& fields
|
||||
.get("schema_version")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
== Some(1)
|
||||
&& fields
|
||||
.get("connection_revision")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(connection.revision.as_str())
|
||||
}
|
||||
|
||||
pub(super) fn timestamp(value: &str) -> Option<SystemTime> {
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.ok()
|
||||
@@ -51,14 +79,15 @@ pub(super) fn component_fresh(
|
||||
}
|
||||
|
||||
pub(super) fn decode(data: &OpaqueProviderData) -> Option<PeerLink> {
|
||||
PeerLink::deserialize(
|
||||
let peer = PeerLink::deserialize(
|
||||
serde::de::value::MapDeserializer::<_, serde_json::Error>::new(
|
||||
data.expose_to_provider()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.as_str(), value)),
|
||||
),
|
||||
)
|
||||
.ok()
|
||||
.ok()?;
|
||||
(peer.object == OBJECT && peer.schema_version == SCHEMA_VERSION).then_some(peer)
|
||||
}
|
||||
|
||||
pub(super) fn parse(
|
||||
@@ -67,11 +96,17 @@ pub(super) fn parse(
|
||||
) -> Result<OpaqueProviderData, UpstreamInfoFailure> {
|
||||
let mut peer: PeerLink =
|
||||
serde_json::from_slice(bytes).map_err(|_| UpstreamInfoFailure::InvalidResponse)?;
|
||||
let valid_text = |value: &str, max: usize| {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
};
|
||||
let observed = timestamp(&peer.observed_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let expires = timestamp(&peer.expires_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
let budget_windows = peer
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.data.as_ref())
|
||||
.map_or(0, |usage| usage.windows.len());
|
||||
if peer.object != OBJECT
|
||||
|| peer.schema_version != SCHEMA_VERSION
|
||||
|| observed > now + Duration::from_secs(60)
|
||||
@@ -79,125 +114,39 @@ pub(super) fn parse(
|
||||
|| expires <= observed
|
||||
|| expires > observed + MAX_AGE
|
||||
|| peer.capabilities.len() > 32
|
||||
|| peer
|
||||
.quota
|
||||
.accounts
|
||||
|| accounts
|
||||
.iter()
|
||||
.map(|account| account.windows.len())
|
||||
.sum::<usize>()
|
||||
> MAX_QUOTA_WINDOWS
|
||||
|| peer.quota.accounts.len() > MAX_ACCOUNTS
|
||||
|| budget_windows > MAX_QUOTA_WINDOWS
|
||||
|| accounts.len() > MAX_ACCOUNTS
|
||||
|| peer.models.len() > MAX_MODELS
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
// 未授权能力不能借同名数据块越过逐层公开边界
|
||||
if !peer.capabilities.iter().any(|value| value == "key_usage") {
|
||||
peer.usage.data = None;
|
||||
let disclosed = |capability| peer.capabilities.iter().any(|value| value == capability);
|
||||
if !disclosed("key_usage") {
|
||||
peer.usage = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "pool_summary")
|
||||
{
|
||||
peer.pool = Default::default();
|
||||
if !disclosed("pool_summary") {
|
||||
peer.pool = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "quota_windows")
|
||||
{
|
||||
peer.quota = Default::default();
|
||||
if !disclosed("quota_windows") {
|
||||
peer.quota = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "model_availability")
|
||||
{
|
||||
if !disclosed("model_availability") {
|
||||
peer.models.clear();
|
||||
}
|
||||
if peer.pool.disclosure == PeerPoolDisclosure::Summary {
|
||||
let count = peer
|
||||
.pool
|
||||
.account_count
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let states = peer
|
||||
.pool
|
||||
.states
|
||||
.as_ref()
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &peer.pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(pool) = &peer.pool {
|
||||
validate_pool(pool)?;
|
||||
}
|
||||
if let Some(usage) = &peer.usage.data {
|
||||
if !valid_text(&usage.unit, 16) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
if [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.resets_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
if let Some(usage) = peer.usage.as_ref().and_then(|usage| usage.data.as_ref()) {
|
||||
validate_usage(usage)?;
|
||||
}
|
||||
for account in &peer.quota.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| !valid_text(&window.label, 256)
|
||||
|| !valid_text(&window.group, 128)
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(quota) = &peer.quota {
|
||||
validate_quota(quota)?;
|
||||
}
|
||||
for model in &peer.models {
|
||||
if !valid_text(&model.model, 256)
|
||||
@@ -219,6 +168,113 @@ pub(super) fn parse(
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_text(value: &str, max: usize) -> bool {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
}
|
||||
|
||||
fn validate_pool(pool: &PeerPool) -> Result<(), UpstreamInfoFailure> {
|
||||
let count = pool.account_count;
|
||||
let states = &pool.states;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_usage(usage: &PeerKeyUsage) -> Result<(), UpstreamInfoFailure> {
|
||||
if usage
|
||||
.unit
|
||||
.as_deref()
|
||||
.is_some_and(|unit| !valid_text(unit, 16))
|
||||
|| !usage.windows.is_empty() && usage.unit.is_none()
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
let mut keys = HashSet::new();
|
||||
for window in &usage.windows {
|
||||
if !valid_text(&window.key, 256)
|
||||
|| !keys.insert(&window.key)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window.window_seconds == Some(0)
|
||||
|| [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_quota(quota: &PeerQuota) -> Result<(), UpstreamInfoFailure> {
|
||||
for account in "a.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window
|
||||
.group
|
||||
.as_deref()
|
||||
.is_some_and(|group| !valid_text(group, 128))
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn decimal(value: &str) -> Option<f64> {
|
||||
if value.is_empty()
|
||||
|| value.len() > 64
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::plugin_update::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use super::system_preflight::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use crate::model::{MutationContext, system::SystemRestartPlan};
|
||||
use async_trait::async_trait;
|
||||
|
||||
|
||||
@@ -1166,6 +1166,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -41,6 +41,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -11,11 +11,11 @@ mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod peer_link;
|
||||
mod plugin_update;
|
||||
mod plugins;
|
||||
mod proxies;
|
||||
mod settings;
|
||||
mod system;
|
||||
mod system_preflight;
|
||||
mod usage_data;
|
||||
|
||||
use std::{
|
||||
|
||||
@@ -1384,6 +1384,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 公共互联按 Provider 声明发现连接,不将认证种类或调度观测绑在 OpenAI 上
|
||||
|
||||
mod refresh;
|
||||
|
||||
use super::super::UnavailableProvider;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
@@ -390,14 +392,13 @@ impl PeerTransport for TestTransport {
|
||||
.push(account.id().as_str().to_owned());
|
||||
let now = Utc::now();
|
||||
Ok(serde_json::to_vec(&serde_json::json!({
|
||||
"object":"cpr.peer", "schema_version":1,
|
||||
"object":"cpr.peer", "schema_version":2,
|
||||
"observed_at":now.to_rfc3339(),
|
||||
"expires_at":(now + chrono::Duration::minutes(5)).to_rfc3339(),
|
||||
"capabilities":["key_usage"],
|
||||
"usage":{"status":"available","observed_at":now.to_rfc3339(),"data":{"unit":"USD","allowed":false,
|
||||
"daily":{"total":"1","used":"1","remaining":"0","resetsAt":(now + chrono::Duration::days(1)).to_rfc3339()},
|
||||
"weekly":{"total":"0","used":"0","remaining":"0"}}},
|
||||
"pool":{"disclosure":"none"}, "quota":{"disclosure":"none"}, "models":[]
|
||||
"windows":[{"key":"daily","total":"1","used":"1","remaining":"0","reset_at":(now + chrono::Duration::days(1)).to_rfc3339()}]}},
|
||||
"models":[]
|
||||
})).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
//! 互联发现结果的持久恢复、连接失效、手动重查与临时失败退避
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::PeerUpstreamBundle;
|
||||
use gateway_core::task::WorkerRegistration;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_survives_restart_until_manual_rediscovery() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
// 入库后的首查与手动刷新使用同一入口,不支持协议也是可保存的正常发现结果
|
||||
let initial = bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(!initial.upstream.unwrap().discovered);
|
||||
let first_worker = worker(&mut bundle);
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&first_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
drop(first_worker);
|
||||
drop(bundle);
|
||||
|
||||
let mut restarted = initialize(accounts, transport.clone());
|
||||
let info = restarted
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert!(!info.discovered);
|
||||
assert_eq!(info.last_error.as_deref(), Some("unsupported"));
|
||||
let resumed_worker = worker(&mut restarted);
|
||||
cycle(&resumed_worker).await;
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
let summary = restarted
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
summary.upstream.unwrap().last_error.as_deref(),
|
||||
Some("unsupported")
|
||||
);
|
||||
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
let discovered = restarted.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(discovered.upstream.unwrap().discovered);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
tokio::time::advance(Duration::from_secs(360)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_keeps_manual_refresh_throttling() {
|
||||
for retry_after in [None, Some(Duration::from_secs(90))] {
|
||||
let (bundle, _, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
*transport.failure.lock().unwrap() = Some(PeerFetchFailure {
|
||||
kind: UpstreamInfoFailure::Unsupported,
|
||||
retry_after,
|
||||
});
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
let delay = retry_after.unwrap_or(Duration::from_secs(15));
|
||||
tokio::time::advance(delay - Duration::from_secs(1)).await;
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.refresh_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn ordinary_edits_keep_detection_but_proxy_and_credential_changes_reprobe() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
let worker = worker(&mut bundle);
|
||||
// 无识别记录的存量账号由周期任务补查
|
||||
cycle(&worker).await;
|
||||
let initial = accounts.accounts.lock().unwrap()[0].clone();
|
||||
let edited = initial.clone().with_notes(Some("修改备注".into()));
|
||||
*accounts.accounts.lock().unwrap() = vec![edited.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
|
||||
let reconnected = edited.with_outbound_proxy(Some(
|
||||
OutboundProxy::parse("http://proxy.invalid:8080").unwrap(),
|
||||
));
|
||||
*accounts.accounts.lock().unwrap() = vec![reconnected.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
|
||||
let replaced = ProviderAccount::new(
|
||||
id.clone(),
|
||||
initial.provider().clone(),
|
||||
"new credential".into(),
|
||||
None,
|
||||
"peer_test".into(),
|
||||
CredentialRevision::new(2).unwrap(),
|
||||
None,
|
||||
)
|
||||
.with_outbound_proxy(reconnected.outbound_proxy().cloned())
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
*accounts.accounts.lock().unwrap() = vec![replaced];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn uncertain_discovery_failures_retry_without_becoming_unsupported() {
|
||||
for failure in [
|
||||
UpstreamInfoFailure::Unauthorized,
|
||||
UpstreamInfoFailure::Unavailable,
|
||||
UpstreamInfoFailure::RateLimited,
|
||||
UpstreamInfoFailure::InvalidResponse,
|
||||
] {
|
||||
let (mut bundle, _, transport, id) = fixture(failure);
|
||||
let worker = worker(&mut bundle);
|
||||
cycle(&worker).await;
|
||||
let info = bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert_eq!(info.last_error.as_deref(), Some(failure.as_str()));
|
||||
assert!(!info.discovered);
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
tokio::time::advance(Duration::from_secs(
|
||||
if failure == UpstreamInfoFailure::Unauthorized {
|
||||
1800
|
||||
} else {
|
||||
30
|
||||
},
|
||||
))
|
||||
.await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(
|
||||
failure: UpstreamInfoFailure,
|
||||
) -> (
|
||||
PeerUpstreamBundle,
|
||||
Arc<TestAccounts>,
|
||||
Arc<DetectionTransport>,
|
||||
ProviderAccountId,
|
||||
) {
|
||||
let record = account("acct_peer", "openai", "peer_test");
|
||||
let id = record.id().clone();
|
||||
let accounts = Arc::new(TestAccounts {
|
||||
accounts: Mutex::new(vec![record]),
|
||||
records: Mutex::default(),
|
||||
failed_provider: None,
|
||||
});
|
||||
let transport = Arc::new(DetectionTransport {
|
||||
requests: AtomicUsize::new(0),
|
||||
failure: Mutex::new(Some(failure.into())),
|
||||
success: TestTransport(Mutex::default()),
|
||||
});
|
||||
(
|
||||
initialize(accounts.clone(), transport.clone()),
|
||||
accounts,
|
||||
transport,
|
||||
id,
|
||||
)
|
||||
}
|
||||
|
||||
fn initialize(
|
||||
accounts: Arc<TestAccounts>,
|
||||
transport: Arc<DetectionTransport>,
|
||||
) -> PeerUpstreamBundle {
|
||||
let mut provider = UnavailableProvider::new("openai");
|
||||
provider.peer_enabled = true;
|
||||
let registry =
|
||||
ProviderAdminRegistry::new([Arc::new(provider) as Arc<dyn ProviderAdmin>]).unwrap();
|
||||
initialize_peer_upstream(accounts, ®istry, transport).unwrap()
|
||||
}
|
||||
|
||||
fn worker(bundle: &mut PeerUpstreamBundle) -> WorkerRegistration {
|
||||
let WorkerContribution::Registration(worker) = bundle.take_worker_contributions().remove(0)
|
||||
else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
worker
|
||||
}
|
||||
|
||||
async fn cycle(worker: &WorkerRegistration) {
|
||||
let WorkerRunnable::Scheduled { task, .. } = &worker.runnable else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
task.run_cycle(WorkerCycleContext::new(
|
||||
worker.id.clone(),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
struct DetectionTransport {
|
||||
requests: AtomicUsize,
|
||||
failure: Mutex<Option<PeerFetchFailure>>,
|
||||
success: TestTransport,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl PeerTransport for DetectionTransport {
|
||||
async fn fetch(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
connection: &PeerConnection,
|
||||
) -> Result<Vec<u8>, PeerFetchFailure> {
|
||||
self.requests.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(failure) = self.failure.lock().unwrap().as_ref() {
|
||||
return Err(PeerFetchFailure {
|
||||
kind: failure.kind,
|
||||
retry_after: failure.retry_after,
|
||||
});
|
||||
}
|
||||
self.success.fetch(account, connection).await
|
||||
}
|
||||
}
|
||||
@@ -334,6 +334,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
pub use crate::time::parse_datetime;
|
||||
|
||||
/// 观测列表默认页大小
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小
|
||||
@@ -256,16 +258,6 @@ impl DiagnosticDimension {
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = trimmed(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
/// 解析 HTTP 状态码
|
||||
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
|
||||
value
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -751,6 +752,7 @@ impl RuntimeSettingsFields {
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: self.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: self.openai_account_affinity,
|
||||
max_account_rotations: self.max_account_rotations,
|
||||
@@ -790,6 +792,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: settings.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
|
||||
@@ -9,25 +9,7 @@ use axum::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// 管理端 wire 字段校验错误;只携带稳定字段名,不回显输入值
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WireValidationError {
|
||||
field: &'static str,
|
||||
}
|
||||
|
||||
impl WireValidationError {
|
||||
/// 构造字段校验错误
|
||||
#[must_use]
|
||||
pub const fn new(field: &'static str) -> Self {
|
||||
Self { field }
|
||||
}
|
||||
|
||||
/// 返回未通过校验的字段名
|
||||
#[must_use]
|
||||
pub const fn field(self) -> &'static str {
|
||||
self.field
|
||||
}
|
||||
}
|
||||
pub use crate::WireValidationError;
|
||||
|
||||
/// 成功响应的稳定业务码
|
||||
pub const ADMIN_OK_CODE: u32 = 200;
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! 数据面与公共 HTTP 中间件共用的客户端错误合同和重试投影
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::{HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// OpenAI 风格的协议错误 JSON
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolErrorBody {
|
||||
/// 错误对象
|
||||
pub error: ProtocolError,
|
||||
}
|
||||
|
||||
/// 不包含请求正文或 prompt 的错误对象
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolError {
|
||||
/// OpenAI 错误类别
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
/// 稳定机器码
|
||||
pub code: &'static str,
|
||||
/// 安全的可读信息
|
||||
pub message: String,
|
||||
/// 出错字段;不包含字段值
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
}
|
||||
|
||||
impl ProtocolErrorBody {
|
||||
/// 序列化为 JSON value
|
||||
#[must_use]
|
||||
pub fn into_value(self) -> Value {
|
||||
serde_json::json!({ "error": self.error })
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
pub fn openai_error_response(
|
||||
status: StatusCode,
|
||||
message: &str,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
) -> (StatusCode, Json<Value>) {
|
||||
let body = json!({
|
||||
"error": {"message": message, "type": error_type, "code": code}
|
||||
});
|
||||
match capacity_error_for_client(&body) {
|
||||
Some(body) => (StatusCode::SERVICE_UNAVAILABLE, Json(body)),
|
||||
None => (status, Json(body)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(crate) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
let capacity_code = |value: &Value| {
|
||||
value
|
||||
.as_str()
|
||||
.is_some_and(|code| client_error_code(code) != code)
|
||||
};
|
||||
if !CODE_PATHS
|
||||
.iter()
|
||||
.any(|path| data.pointer(path).is_some_and(capacity_code))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
{
|
||||
*code = Value::String("server_error".to_owned());
|
||||
}
|
||||
}
|
||||
// WS 包装错误按 HTTP 状态分类;保留 400/429 会使客户端终止重试
|
||||
for field in ["status", "status_code"] {
|
||||
if let Some(status) = projected.get_mut(field)
|
||||
&& status.is_number()
|
||||
{
|
||||
*status = json!(503);
|
||||
}
|
||||
}
|
||||
Some(projected)
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
let mut response = openai_error_response(
|
||||
status,
|
||||
error.client_message(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(delay) = error.retry_after()
|
||||
&& let Ok(value) = HeaderValue::from_str(
|
||||
&delay
|
||||
.as_secs()
|
||||
.saturating_add(u64::from(delay.subsec_nanos() > 0))
|
||||
.max(1)
|
||||
.to_string(),
|
||||
)
|
||||
{
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
kind: GatewayErrorKind,
|
||||
) -> (StatusCode, &'static str, &'static str) {
|
||||
match kind {
|
||||
GatewayErrorKind::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
),
|
||||
GatewayErrorKind::Unsupported => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"unsupported_capability",
|
||||
),
|
||||
GatewayErrorKind::Unauthorized => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
),
|
||||
GatewayErrorKind::PolicyDenied => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"invalid_request_error",
|
||||
"policy_denied",
|
||||
),
|
||||
GatewayErrorKind::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
),
|
||||
GatewayErrorKind::NoAvailableProvider => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_full",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_timeout",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
),
|
||||
GatewayErrorKind::BudgetExhausted => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"usage_limit_reached",
|
||||
"usage_limit_reached",
|
||||
),
|
||||
GatewayErrorKind::UpstreamUnavailable => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
"request_timeout",
|
||||
),
|
||||
GatewayErrorKind::Cancelled => (
|
||||
StatusCode::REQUEST_TIMEOUT,
|
||||
"server_error",
|
||||
"request_cancelled",
|
||||
),
|
||||
GatewayErrorKind::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ use crate::openai::service::OpenAiService;
|
||||
|
||||
pub mod admin;
|
||||
pub mod auth;
|
||||
mod client_error;
|
||||
mod health;
|
||||
mod key_usage;
|
||||
mod middleware;
|
||||
@@ -40,6 +41,26 @@ mod time;
|
||||
|
||||
pub use time::{RequestBucketView, TimePresenter};
|
||||
|
||||
/// HTTP wire 字段校验错误;只携带稳定字段名,不回显输入值
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WireValidationError {
|
||||
field: &'static str,
|
||||
}
|
||||
|
||||
impl WireValidationError {
|
||||
/// 构造字段校验错误
|
||||
#[must_use]
|
||||
pub const fn new(field: &'static str) -> Self {
|
||||
Self { field }
|
||||
}
|
||||
|
||||
/// 返回未通过校验的字段名
|
||||
#[must_use]
|
||||
pub const fn field(self) -> &'static str {
|
||||
self.field
|
||||
}
|
||||
}
|
||||
|
||||
/// API-owned HTTP 与静态资源配置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct ApiConfig {
|
||||
|
||||
@@ -262,7 +262,7 @@ async fn handle(
|
||||
Err(MiddlewareError::Gateway(error))
|
||||
if error.kind() == GatewayErrorKind::MiddlewareCapacityUnavailable =>
|
||||
{
|
||||
crate::openai::error::gateway_error_response(&error)
|
||||
crate::client_error::gateway_error_response(&error)
|
||||
}
|
||||
Err(error) => {
|
||||
let status = if error.is_rejected() {
|
||||
|
||||
@@ -15,7 +15,8 @@ use gateway_core::{
|
||||
use gateway_protocol::openai::sse::{DONE_SSE_FRAME, SseEvent, SseEventDecoder, sse_frame_is_done};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::openai::{middleware::error_response, responses::ProtocolErrorBody};
|
||||
use crate::client_error::ProtocolErrorBody;
|
||||
use crate::openai::middleware::error_response;
|
||||
|
||||
use super::response::{ChatIdentity, ChatStreamEncoder, project_response};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use gateway_protocol::openai::sse::encode_sse_event;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::openai::responses::{ProtocolError, ProtocolErrorBody};
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(in crate::openai) struct ChatIdentity {
|
||||
|
||||
@@ -16,6 +16,7 @@ use gateway_core::{
|
||||
operation::Operation,
|
||||
};
|
||||
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
use crate::middleware::headers::encode_headers;
|
||||
use crate::openai::middleware::{
|
||||
ExpectedBody, PendingExecution, RequestInput, buffered_response, error_response,
|
||||
@@ -25,7 +26,6 @@ use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
use super::{
|
||||
error::{engine_error_response, gateway_error_response, protocol_error_response},
|
||||
responses::{ProtocolError, ProtocolErrorBody},
|
||||
service::OpenAiService,
|
||||
};
|
||||
|
||||
|
||||
@@ -8,72 +8,18 @@ use axum::{
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::EngineError,
|
||||
error::{GatewayError, GatewayErrorKind, ProviderErrorKind},
|
||||
error::ProviderErrorKind,
|
||||
policy::{ClientVersionRejection, CodexClientKind},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
|
||||
use super::responses::ProtocolErrorBody;
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client};
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
pub fn openai_error_response(
|
||||
status: StatusCode,
|
||||
message: &str,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
) -> (StatusCode, Json<Value>) {
|
||||
let body = json!({
|
||||
"error": {"message": message, "type": error_type, "code": code}
|
||||
});
|
||||
match capacity_error_for_client(&body) {
|
||||
Some(body) => (StatusCode::SERVICE_UNAVAILABLE, Json(body)),
|
||||
None => (status, Json(body)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(super) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(super) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
let capacity_code = |value: &Value| {
|
||||
value
|
||||
.as_str()
|
||||
.is_some_and(|code| client_error_code(code) != code)
|
||||
};
|
||||
if !CODE_PATHS
|
||||
.iter()
|
||||
.any(|path| data.pointer(path).is_some_and(capacity_code))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
{
|
||||
*code = Value::String("server_error".to_owned());
|
||||
}
|
||||
}
|
||||
// WS 包装错误按 HTTP 状态分类;保留 400/429 会使客户端终止重试
|
||||
for field in ["status", "status_code"] {
|
||||
if let Some(status) = projected.get_mut(field)
|
||||
&& status.is_number()
|
||||
{
|
||||
*status = json!(503);
|
||||
}
|
||||
}
|
||||
Some(projected)
|
||||
}
|
||||
pub use crate::client_error::{
|
||||
gateway_error_contract, gateway_error_response, openai_error_response,
|
||||
};
|
||||
|
||||
/// 严格协议 decoder/encoder 返回的安全错误 body
|
||||
pub fn protocol_error_response(status: StatusCode, body: ProtocolErrorBody) -> Response {
|
||||
@@ -151,32 +97,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
)
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
let mut response = openai_error_response(
|
||||
status,
|
||||
error.client_message(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(delay) = error.retry_after()
|
||||
&& let Ok(value) = HeaderValue::from_str(
|
||||
&delay
|
||||
.as_secs()
|
||||
.saturating_add(u64::from(delay.subsec_nanos() > 0))
|
||||
.max(1)
|
||||
.to_string(),
|
||||
)
|
||||
{
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// 在下游尚未提交时交付 Provider 的 HTTP 失败响应,并应用客户端容量恢复合同
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let capacity_unavailable = matches!(error, EngineError::Provider(error)
|
||||
@@ -233,107 +153,3 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
kind: GatewayErrorKind,
|
||||
) -> (StatusCode, &'static str, &'static str) {
|
||||
match kind {
|
||||
GatewayErrorKind::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
),
|
||||
GatewayErrorKind::Unsupported => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"unsupported_capability",
|
||||
),
|
||||
GatewayErrorKind::Unauthorized => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
),
|
||||
GatewayErrorKind::PolicyDenied => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"invalid_request_error",
|
||||
"policy_denied",
|
||||
),
|
||||
GatewayErrorKind::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
),
|
||||
GatewayErrorKind::NoAvailableProvider => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_full",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_timeout",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
),
|
||||
GatewayErrorKind::BudgetExhausted => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"usage_limit_reached",
|
||||
"usage_limit_reached",
|
||||
),
|
||||
GatewayErrorKind::UpstreamUnavailable => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
"request_timeout",
|
||||
),
|
||||
GatewayErrorKind::Cancelled => (
|
||||
StatusCode::REQUEST_TIMEOUT,
|
||||
"server_error",
|
||||
"request_cancelled",
|
||||
),
|
||||
GatewayErrorKind::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,34 +5,31 @@ use gateway_admin::model::peer_link::{PeerLinkExport, PeerModelState as ModelSta
|
||||
use gateway_core::metering::Decimal;
|
||||
use gateway_protocol::peer::{
|
||||
OBJECT, PeerBudgetWindow, PeerComponentStatus, PeerKeyUsage, PeerLink, PeerModelAvailability,
|
||||
PeerModelCoverage, PeerModelState, PeerPlanCount, PeerPool, PeerPoolDisclosure, PeerPoolStates,
|
||||
PeerQuota, PeerQuotaAccount, PeerQuotaDisclosure, PeerQuotaWindow, PeerUsage, SCHEMA_VERSION,
|
||||
PeerModelCoverage, PeerModelState, PeerPlanCount, PeerPool, PeerPoolStates, PeerQuota,
|
||||
PeerQuotaAccount, PeerQuotaWindow, PeerUsage, SCHEMA_VERSION,
|
||||
};
|
||||
use std::time::SystemTime;
|
||||
|
||||
pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
let mut capabilities = vec!["key_usage".to_owned()];
|
||||
let pool = export.pool.map_or_else(PeerPool::default, |pool| {
|
||||
let pool = export.pool.map(|pool| {
|
||||
capabilities.push("pool_summary".to_owned());
|
||||
PeerPool {
|
||||
disclosure: PeerPoolDisclosure::Summary,
|
||||
observed_at: Some(export.observed_at.to_rfc3339()),
|
||||
account_count: Some(
|
||||
pool.ready
|
||||
+ pool.cooldown
|
||||
+ pool.exhausted
|
||||
+ pool.disabled
|
||||
+ pool.error
|
||||
+ pool.unknown,
|
||||
),
|
||||
states: Some(PeerPoolStates {
|
||||
account_count: pool.ready
|
||||
+ pool.cooldown
|
||||
+ pool.exhausted
|
||||
+ pool.disabled
|
||||
+ pool.error
|
||||
+ pool.unknown,
|
||||
states: PeerPoolStates {
|
||||
ready: pool.ready,
|
||||
cooldown: pool.cooldown,
|
||||
exhausted: pool.exhausted,
|
||||
disabled: pool.disabled,
|
||||
error: pool.error,
|
||||
unknown: pool.unknown,
|
||||
}),
|
||||
},
|
||||
plans: pool.plans.map(|plans| {
|
||||
plans
|
||||
.into_iter()
|
||||
@@ -41,10 +38,9 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
}),
|
||||
}
|
||||
});
|
||||
let quota = export.quota.map_or_else(PeerQuota::default, |accounts| {
|
||||
let quota = export.quota.map(|accounts| {
|
||||
capabilities.extend(["quota_windows".to_owned(), "model_availability".to_owned()]);
|
||||
PeerQuota {
|
||||
disclosure: PeerQuotaDisclosure::Accounts,
|
||||
accounts: accounts
|
||||
.into_iter()
|
||||
.map(|account| PeerQuotaAccount {
|
||||
@@ -57,8 +53,8 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
.into_iter()
|
||||
.map(|window| PeerQuotaWindow {
|
||||
key: window.key,
|
||||
group: window.group,
|
||||
label: window.label,
|
||||
group: Some(window.group),
|
||||
label: Some(window.label),
|
||||
limit_id: window.limit_id,
|
||||
limit_name: window.limit_name,
|
||||
role: window.role.map(|role| role.as_str().to_owned()),
|
||||
@@ -78,26 +74,34 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
capabilities,
|
||||
observed_at: export.observed_at.to_rfc3339(),
|
||||
expires_at: export.expires_at.to_rfc3339(),
|
||||
usage: PeerUsage {
|
||||
usage: Some(PeerUsage {
|
||||
status: PeerComponentStatus::Available,
|
||||
observed_at: Some(export.budget_observed_at.to_rfc3339()),
|
||||
data: Some(PeerKeyUsage {
|
||||
unit: "USD".to_owned(),
|
||||
unit: Some("USD".to_owned()),
|
||||
allowed: Some(export.budget_access.allowed()),
|
||||
daily: budget_window(
|
||||
export.budget.limits.daily_usd,
|
||||
export.budget.daily_used_usd,
|
||||
export.budget.daily_resets_at,
|
||||
export.budget_access.daily_limit_reached,
|
||||
),
|
||||
weekly: budget_window(
|
||||
export.budget.limits.weekly_usd,
|
||||
export.budget.weekly_used_usd,
|
||||
export.budget.weekly_resets_at,
|
||||
export.budget_access.weekly_limit_reached,
|
||||
),
|
||||
windows: vec![
|
||||
budget_window(
|
||||
"daily",
|
||||
"日额度",
|
||||
86400,
|
||||
export.budget.limits.daily_usd,
|
||||
export.budget.daily_used_usd,
|
||||
export.budget.daily_resets_at,
|
||||
export.budget_access.daily_limit_reached,
|
||||
),
|
||||
budget_window(
|
||||
"weekly",
|
||||
"周额度",
|
||||
604800,
|
||||
export.budget.limits.weekly_usd,
|
||||
export.budget.weekly_used_usd,
|
||||
export.budget.weekly_resets_at,
|
||||
export.budget_access.weekly_limit_reached,
|
||||
),
|
||||
],
|
||||
}),
|
||||
},
|
||||
}),
|
||||
pool,
|
||||
quota,
|
||||
models: export
|
||||
@@ -123,6 +127,9 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
}
|
||||
|
||||
fn budget_window(
|
||||
key: &str,
|
||||
label: &str,
|
||||
window_seconds: u64,
|
||||
total: Decimal,
|
||||
used: Decimal,
|
||||
resets_at: Option<SystemTime>,
|
||||
@@ -130,10 +137,13 @@ fn budget_window(
|
||||
) -> PeerBudgetWindow {
|
||||
let limited = total != Decimal::ZERO;
|
||||
PeerBudgetWindow {
|
||||
key: key.to_owned(),
|
||||
label: Some(label.to_owned()),
|
||||
window_seconds: Some(window_seconds),
|
||||
limit_reached: Some(limit_reached),
|
||||
total: limited.then(|| total.canonical()),
|
||||
used: used.canonical(),
|
||||
remaining: limited.then(|| total.saturating_sub(used).canonical()),
|
||||
resets_at: resets_at.map(|time| DateTime::<Utc>::from(time).to_rfc3339()),
|
||||
reset_at: resets_at.map(|time| DateTime::<Utc>::from(time).to_rfc3339()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,39 +1,8 @@
|
||||
//! OpenAI Responses adapter 的稳定错误 contract
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
use thiserror::Error;
|
||||
|
||||
/// OpenAI 风格的协议错误 JSON
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolErrorBody {
|
||||
/// 错误对象
|
||||
pub error: ProtocolError,
|
||||
}
|
||||
|
||||
/// 不包含请求正文或 prompt 的错误对象
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolError {
|
||||
/// OpenAI 错误类别
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
/// 稳定机器码
|
||||
pub code: &'static str,
|
||||
/// 安全的可读信息
|
||||
pub message: String,
|
||||
/// 出错字段;不包含字段值
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
}
|
||||
|
||||
impl ProtocolErrorBody {
|
||||
/// 序列化为 JSON value
|
||||
#[must_use]
|
||||
pub fn into_value(self) -> Value {
|
||||
serde_json::json!({ "error": self.error })
|
||||
}
|
||||
}
|
||||
|
||||
/// Responses 请求解码错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum RequestDecodeError {
|
||||
|
||||
@@ -943,7 +943,7 @@ impl StreamingExecutionBody {
|
||||
Bytes::from(response_failed_sse_event_with_id(
|
||||
self.encoder.response_id(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
super::super::error::client_error_code(
|
||||
crate::client_error::client_error_code(
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
),
|
||||
error.client_message(),
|
||||
|
||||
@@ -8,8 +8,9 @@ mod response;
|
||||
mod validation;
|
||||
pub mod websocket;
|
||||
|
||||
pub use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
pub(crate) use crate::request_context::request_client_context;
|
||||
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub use error::{RequestDecodeError, ResponseEncodeError};
|
||||
pub(in crate::openai) use http::{HttpResponseAdapter, execute_prepared_response};
|
||||
pub(crate) use http::{
|
||||
ResponseAuthorization, ResponsesHttpRequest, execute_prepared_responses, responses,
|
||||
|
||||
@@ -10,7 +10,7 @@ use gateway_protocol::openai::sse::{
|
||||
use serde_json::Value;
|
||||
|
||||
use super::error::ResponseEncodeError;
|
||||
use crate::openai::error::capacity_error_for_client;
|
||||
use crate::client_error::capacity_error_for_client;
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Responses WebSocket 入站请求与下行事件的纯协议映射
|
||||
|
||||
use crate::client_error::capacity_error_for_client;
|
||||
|
||||
use axum::http::{HeaderName, HeaderValue, StatusCode};
|
||||
use gateway_core::engine::EngineError;
|
||||
use gateway_core::error::ProviderErrorKind;
|
||||
@@ -7,9 +9,7 @@ use gateway_core::event::ProviderResponseHeader;
|
||||
use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::openai::error::{
|
||||
capacity_error_for_client, gateway_error_contract, gateway_error_from_engine,
|
||||
};
|
||||
use crate::openai::error::{gateway_error_contract, gateway_error_from_engine};
|
||||
|
||||
use super::super::{
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError,
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
//! 管理与自助页面共用的时间展示投影
|
||||
//! 管理与自助页面共用的时间查询解析与展示投影
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::time::DeploymentTimeZone;
|
||||
|
||||
use crate::WireValidationError;
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn query_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
@@ -10,8 +22,7 @@ pub(crate) fn query_range(
|
||||
as_of: Option<i64>,
|
||||
timezone: DeploymentTimeZone,
|
||||
default_period: &str,
|
||||
) -> Result<gateway_admin::model::observability::TimeRange, crate::admin::WireValidationError> {
|
||||
use crate::admin::{WireValidationError, observability::parse_datetime};
|
||||
) -> Result<gateway_admin::model::observability::TimeRange, WireValidationError> {
|
||||
use gateway_admin::model::observability::{CalendarPeriod, TimeRange};
|
||||
if start.is_some() || end.is_some() {
|
||||
if period.is_some() || as_of.is_some() {
|
||||
|
||||
@@ -851,35 +851,21 @@ mod actions {
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"data": {},
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_complete_should_not_accept_group_assignment() {
|
||||
fn oauth_complete_rejects_unknown_fields() {
|
||||
let flow_id = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7_u8; 32]);
|
||||
let request: CompleteAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
let mut payload = json!({
|
||||
"provider": "openai",
|
||||
"flowId": flow_id,
|
||||
"callbackUrl": "http://localhost/callback?code=ok"
|
||||
}))
|
||||
.expect("decode OAuth completion");
|
||||
});
|
||||
let request: CompleteAccountAuthorizationRequest =
|
||||
serde_json::from_value(payload.clone()).expect("decode OAuth completion");
|
||||
assert!(request.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<CompleteAccountAuthorizationRequest>(json!({
|
||||
"provider": "xai",
|
||||
"flowId": "flow_test",
|
||||
"callbackUrl": "http://localhost/callback?code=ok",
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
payload["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<CompleteAccountAuthorizationRequest>(payload).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -651,6 +651,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
max_waiting_per_key: command.values.max_waiting_per_key,
|
||||
max_waiting_per_account: command.values.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.values.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: command.values.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: command
|
||||
.values
|
||||
.openai_guardian_reserved_concurrency,
|
||||
@@ -1893,6 +1894,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -879,7 +879,7 @@ async fn usage_route_should_forward_name_and_model_contains_filters() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
async fn usage_route_projects_request_facts_and_billing() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
@@ -1123,7 +1123,6 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
"userAgent": "codex-cli/1.0",
|
||||
})
|
||||
);
|
||||
assert!(value["data"]["items"][0].get("metadata").is_none());
|
||||
assert_eq!(
|
||||
value["data"]["items"][1]["billing"]["totalAmountDisplay"],
|
||||
"≈ $0.007"
|
||||
@@ -1731,7 +1730,7 @@ async fn usage_status_query_accepts_three_modes_and_rejects_unknown_values() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_identify_metrics_without_localized_labels() {
|
||||
async fn dashboard_summary_identifies_trend_metrics_and_account_usage_values() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
@@ -1884,7 +1883,6 @@ async fn dashboard_summary_should_identify_metrics_without_localized_labels() {
|
||||
assert_eq!(summary.len(), metrics.len());
|
||||
for (item, metric) in summary.iter().zip(metrics) {
|
||||
assert_eq!(item["metric"], metric);
|
||||
assert!(item.get("label").is_none());
|
||||
assert!(item["value"].is_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ fn update_body() -> Value {
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"maxRetryAfterSeconds": 240,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
@@ -426,112 +427,20 @@ fn settings_request_should_reject_non_semver_client_min() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
fn settings_response_formats_update_time_in_deployment_timezone() {
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::account::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
PublicModelId::new("gpt-5.4").expect("public model"),
|
||||
UpstreamModelId::new("gpt-5.5").expect("upstream model"),
|
||||
),
|
||||
(
|
||||
PublicModelId::new("grok-latest").expect("public model"),
|
||||
UpstreamModelId::new("grok-4.5").expect("upstream model"),
|
||||
),
|
||||
]),
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
.expect("timestamp"),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_error_protection: Default::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
};
|
||||
|
||||
let mut settings = super::test_runtime_settings();
|
||||
settings.updated_at = Utc.with_ymd_and_hms(2026, 8, 2, 10, 30, 0).unwrap();
|
||||
let value = serde_json::to_value(RuntimeSettingsView::from((
|
||||
settings,
|
||||
gateway_api::TimePresenter::new(Default::default()),
|
||||
)))
|
||||
.expect("serialize view");
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"configRevision": 7,
|
||||
"providerRequestProfiles": {},
|
||||
"openaiClientProfile": null,
|
||||
"xaiClientProfile": null,
|
||||
"codexPrivacyPolicy": {"enabled":false,"onError":"skip_rule","rules":[]},
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
},
|
||||
"refreshMarginSeconds": 1800,
|
||||
"refreshConcurrency": 4,
|
||||
"maxConcurrentPerAccount": 5,
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"openaiGuardianReservedConcurrency": 0,
|
||||
"openaiAccountAffinity": "relaxed",
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"smartSchedulingDefaults": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 31,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountErrorProtection": gateway_core::account::AccountErrorProtection::default(),
|
||||
"accountWarmupEnabled": false,
|
||||
"accountWarmupScheduleTime": "08:00",
|
||||
"accountWarmupModel": null,
|
||||
"resetCreditLeadMinutes": 10,
|
||||
"resetCreditMaxWeeklyRemainingPercent": null,
|
||||
"updatedAt": "2026-08-02T10:30:00Z",
|
||||
"updatedAtDisplay": "2026-08-02 18:30:00"
|
||||
})
|
||||
);
|
||||
|
||||
assert_eq!(value["updatedAt"], "2026-08-02T10:30:00Z");
|
||||
assert_eq!(value["updatedAtDisplay"], "2026-08-02 18:30:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -243,10 +243,10 @@ async fn exports_scoped_observations_even_when_the_key_budget_is_exhausted_witho
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["object"], "cpr.peer");
|
||||
assert_eq!(body["schema_version"], 1);
|
||||
assert_eq!(body["usage"]["data"]["daily"]["remaining"], "0");
|
||||
assert_eq!(body["schema_version"], 2);
|
||||
assert_eq!(body["usage"]["data"]["windows"][0]["remaining"], "0");
|
||||
assert_eq!(body["usage"]["data"]["allowed"], false);
|
||||
assert_eq!(body["usage"]["data"]["daily"]["limit_reached"], true);
|
||||
assert_eq!(body["usage"]["data"]["windows"][0]["limit_reached"], true);
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert!(body["pool"].get("plans").is_none());
|
||||
assert_eq!(
|
||||
@@ -261,6 +261,114 @@ async fn exports_scoped_observations_even_when_the_key_budget_is_exhausted_witho
|
||||
assert!(fixture.observations.lock().unwrap().summaries.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_plans_use_account_plan_fallback_independently_of_quota_sharing() {
|
||||
for share_quota in [true, false] {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
for (account_plan, quota_plan, expected) in [
|
||||
(None, Some("free"), Some("Free")),
|
||||
(Some("unknown"), Some("free"), Some("Free")),
|
||||
(Some(" "), Some("free"), Some("Free")),
|
||||
(Some("edu_pro"), Some("edu"), Some("EduPro")),
|
||||
(Some("free"), None, Some("Free")),
|
||||
(None, None, None),
|
||||
(None, Some("unknown"), None),
|
||||
] {
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = account_plan.map(str::to_owned);
|
||||
fixture
|
||||
.provider_quota
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.plan_type = quota_plan.map(str::to_owned);
|
||||
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
let plans =
|
||||
expected.map_or_else(|| json!([]), |name| json!([{ "name": name, "count": 1 }]));
|
||||
assert_eq!(
|
||||
body["pool"]["plans"], plans,
|
||||
"{account_plan:?} / {quota_plan:?}"
|
||||
);
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body.get("quota").is_some(), share_quota);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type
|
||||
.as_deref(),
|
||||
account_plan,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_only_summary_limits_exported_data_and_preserves_unknown_on_quota_read_failure() {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota: false,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = None;
|
||||
{
|
||||
let mut quota = fixture.provider_quota.lock().unwrap();
|
||||
let quota = quota.as_mut().unwrap();
|
||||
quota.plan_type = Some("free".into());
|
||||
quota.windows =
|
||||
vec![quota.windows[0].clone(); gateway_protocol::peer::MAX_QUOTA_WINDOWS + 1];
|
||||
}
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(
|
||||
body["pool"]["plans"],
|
||||
json!([{ "name": "Free", "count": 1 }])
|
||||
);
|
||||
assert_eq!(body["capabilities"], json!(["key_usage", "pool_summary"]));
|
||||
assert!(body.get("quota").is_none());
|
||||
|
||||
*fixture.provider_quota.lock().unwrap() = None;
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body["pool"]["plans"], json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requires_authentication_explicit_sharing_and_bound_groups_and_rejects_query_overrides() {
|
||||
let (fixture, app) = fixture(sharing(), RAW_KEY).await;
|
||||
@@ -338,7 +446,7 @@ async fn current_field_permissions_and_direct_account_facts_narrow_the_frozen_sc
|
||||
.peer_sharing
|
||||
.share_quota = false;
|
||||
let body = response_json(query(&app, PATH, Some(RAW_KEY)).await).await;
|
||||
assert_eq!(body["quota"], json!({ "disclosure": "none" }));
|
||||
assert!(body.get("quota").is_none());
|
||||
assert!(body.get("models").is_none());
|
||||
fixture
|
||||
.account
|
||||
@@ -511,7 +619,7 @@ async fn heavy_snapshots_reject_excess_concurrency_and_release_slots_on_completi
|
||||
.await
|
||||
.expect("budget-only queries do not consume heavy-query slots");
|
||||
assert_eq!(budget.status(), StatusCode::OK);
|
||||
assert_eq!(response_json(budget).await["pool"]["disclosure"], "none");
|
||||
assert!(response_json(budget).await.get("pool").is_none());
|
||||
let cancelled = requests.pop().unwrap();
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.unwrap_err().is_cancelled());
|
||||
|
||||
@@ -1677,6 +1677,10 @@ where
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
// 服务器建议不可截短;超出请求冻结的上限后沿用安全换号或原错误出口
|
||||
let retry_after_exceeds_limit = error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after());
|
||||
// Provider 的有界同账号传输恢复是尽力重试策略,不要求把 Sent/Ambiguous
|
||||
// 改写成安全证明;外部副作用、取消、预算与下游提交仍由 Core 拦截
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
@@ -1699,7 +1703,8 @@ where
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if !execution_effect_observed
|
||||
}) if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1707,7 +1712,8 @@ where
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if !execution_effect_observed
|
||||
if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1753,7 +1759,7 @@ where
|
||||
max_delay,
|
||||
jitter,
|
||||
..
|
||||
}) if ordinary_retry && !suppress_transient_retry => {
|
||||
}) if ordinary_retry && !suppress_transient_retry && !retry_after_exceeds_limit => {
|
||||
let retries = self
|
||||
.transient_retry_counts
|
||||
.entry(current.metadata.provider_account_id().clone())
|
||||
@@ -1832,6 +1838,8 @@ where
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"retryAfterExceeded": retry_after_exceeds_limit,
|
||||
"maxRetryAfterSeconds": self.plan.max_retry_after().as_secs(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
"connectionRetry": connection_retry_requested,
|
||||
|
||||
@@ -576,6 +576,7 @@ pub struct RoutingPlan {
|
||||
account_error_protection: Arc<crate::account::CompiledAccountErrorProtection>,
|
||||
account_error_protection_revision: u64,
|
||||
request_body_limit: usize,
|
||||
max_retry_after: std::time::Duration,
|
||||
pricing: Arc<crate::metering::PricingOverrides>,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
privacy: Option<Arc<dyn crate::settings::privacy::CompiledPrivacyPolicy>>,
|
||||
@@ -598,6 +599,12 @@ impl RoutingPlan {
|
||||
Arc::clone(&self.account_error_protection)
|
||||
}
|
||||
|
||||
/// 本次请求接受的单次 Retry-After 等待上限
|
||||
#[must_use]
|
||||
pub const fn max_retry_after(&self) -> std::time::Duration {
|
||||
self.max_retry_after
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> usize {
|
||||
self.request_body_limit
|
||||
|
||||
@@ -1408,6 +1408,9 @@ impl RuntimeSnapshot {
|
||||
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
@@ -1495,6 +1498,9 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
|
||||
@@ -21,6 +21,7 @@ pub struct SettingsValues {
|
||||
pub(crate) max_waiting_per_key: u32,
|
||||
pub(crate) max_waiting_per_account: u32,
|
||||
pub(crate) concurrency_wait_timeout_seconds: u32,
|
||||
pub(crate) max_retry_after_seconds: u32,
|
||||
pub(crate) openai_guardian_reserved_concurrency: u32,
|
||||
pub(crate) openai_account_affinity: crate::account::AccountAffinity,
|
||||
pub(crate) max_account_rotations: u32,
|
||||
@@ -84,6 +85,13 @@ impl SettingsValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// 网关接受的单次服务器建议等待上限,超过时交由既有换号或错误出口处理
|
||||
#[must_use]
|
||||
pub const fn with_max_retry_after_seconds(mut self, seconds: u32) -> Self {
|
||||
self.max_retry_after_seconds = seconds;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
|
||||
self.max_account_rotations = rotations;
|
||||
@@ -178,6 +186,7 @@ impl SettingsValues {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: crate::account::AccountAffinity::default(),
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -1174,6 +1174,7 @@ fn plan_with_profiles(
|
||||
fast_mode,
|
||||
profiles,
|
||||
Default::default(),
|
||||
240,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1184,6 +1185,7 @@ fn plan_with_protection(
|
||||
fast_mode: FastMode,
|
||||
profiles: BTreeMap<ProviderKind, gateway_core::account::OpaqueProviderData>,
|
||||
protection: gateway_core::account::AccountErrorProtection,
|
||||
max_retry_after_seconds: u32,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
@@ -1237,7 +1239,8 @@ fn plan_with_protection(
|
||||
.unwrap(),
|
||||
)
|
||||
.with_request_location(request_location, true)
|
||||
.with_account_error_protection(protection),
|
||||
.with_account_error_protection(protection)
|
||||
.with_max_retry_after_seconds(max_retry_after_seconds),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -4216,6 +4219,177 @@ fn server_retry_advice_should_override_local_backoff_and_transport_fallback() {
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_after_plan(operation: &Operation, limit: u32) -> RoutingPlan {
|
||||
plan_with_protection(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::MIN),
|
||||
Default::default(),
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
limit,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_at_or_below_the_configured_limit_keeps_the_full_wait() {
|
||||
for (limit, seconds) in [(240, 240), (60, 60), (400, 308)] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transient_rejection().with_retry_after(Duration::from_secs(seconds))
|
||||
)],
|
||||
}]);
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(600)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
cancellation.clone(),
|
||||
))
|
||||
.unwrap();
|
||||
let trace = session.trace().clone();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(block_on(async { futures::poll!(&mut collection) }).is_pending());
|
||||
let trace = trace.snapshot().unwrap();
|
||||
let retry = trace["events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "retry.decided")
|
||||
.unwrap();
|
||||
assert_eq!(retry["data"]["delayMs"], seconds * 1000);
|
||||
assert_eq!(retry["data"]["retryAfterExceeded"], false);
|
||||
cancellation.cancel();
|
||||
assert!(matches!(block_on(collection), Err(EngineError::Cancelled)));
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_above_the_configured_limit_rotates_without_waiting() {
|
||||
for (limit, seconds) in [(240, 241), (240, 308), (60, 61), (0, 1)] {
|
||||
for rejection in [
|
||||
transient_rejection(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_fallback(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(seconds)),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection.with_retry_after(Duration::from_secs(seconds)))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(matches!(
|
||||
block_on(async { futures::poll!(&mut collection) }),
|
||||
std::task::Poll::Ready(Ok(_))
|
||||
));
|
||||
drop(collection);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(contexts[1].required_account(), None);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excessive_retry_after_preserves_the_error_when_rotation_is_unavailable_or_disallowed() {
|
||||
for (rejection, rotation_allowed) in [
|
||||
(transient_rejection(), true),
|
||||
(
|
||||
transient_rejection().with_transient_retry(
|
||||
NonZeroU32::MIN,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
),
|
||||
false,
|
||||
),
|
||||
(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(308)),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let original = Bytes::from_static(
|
||||
br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"308"}}}"#,
|
||||
);
|
||||
let expected_kind = rejection.kind();
|
||||
let rejection = rejection
|
||||
.with_retry_after(Duration::from_secs(308))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/json".to_vec()),
|
||||
original.clone(),
|
||||
));
|
||||
let mut scripts = vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection)],
|
||||
}];
|
||||
if rotation_allowed {
|
||||
scripts.push(Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)));
|
||||
}
|
||||
let (coordinator, _, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let result = block_on(async { futures::poll!(&mut collection) });
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) = result else {
|
||||
panic!("must return the original failure without waiting")
|
||||
};
|
||||
assert_eq!(error.kind(), expected_kind);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(308)));
|
||||
assert_eq!(
|
||||
error.client_visible_upstream_response().unwrap().body(),
|
||||
&original
|
||||
);
|
||||
assert_eq!(
|
||||
provider.contexts.lock().unwrap().len(),
|
||||
if rotation_allowed { 2 } else { 1 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
for policy in [
|
||||
@@ -4223,7 +4397,7 @@ fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let route_plan = retry_after_plan(&operation, 0);
|
||||
let (coordinator, _, _) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
@@ -4944,7 +5118,9 @@ fn structural_event_before_replay_safe_failure_should_switch_account_before_comm
|
||||
|
||||
#[test]
|
||||
fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery() {
|
||||
for close_window in [false, true] {
|
||||
for (close_window, excessive_retry_after) in
|
||||
[(false, false), (true, false), (false, true), (true, true)]
|
||||
{
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let prelude = || {
|
||||
@@ -4971,11 +5147,12 @@ fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery(
|
||||
// 后续前导不能撤销已经交付未知事件的事实
|
||||
items.push(Ok(prelude()));
|
||||
}
|
||||
items.push(Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe()));
|
||||
items.push(Err(if excessive_retry_after {
|
||||
transient_rejection().with_retry_after(Duration::from_secs(308))
|
||||
} else {
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
|
||||
.with_replay_safe()
|
||||
}));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
@@ -7185,6 +7362,7 @@ fn error_protection_tracks_real_retry_attempts_and_skips_diagnostic_runs() {
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
policy().as_ref().clone(),
|
||||
240,
|
||||
);
|
||||
let feedback = Arc::new(Feedback::default());
|
||||
let failure = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use crate::system_update::SystemUpdateConfig;
|
||||
use crate::{ConfigError, system_update::SystemUpdateConfig};
|
||||
|
||||
const CONFIG_RELATIVE_PATH: &str = "deploy/config.yaml";
|
||||
const SERVER_HOST_ENV: &str = "CPR_SERVER_HOST";
|
||||
@@ -216,26 +216,6 @@ pub struct FileLoggingConfig {
|
||||
pub max_file_size_mb: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("配置文件 {path} 的字段 {field:?} 不合法")]
|
||||
InvalidValue { path: PathBuf, field: String },
|
||||
#[error("current directory is unavailable")]
|
||||
CurrentDirectory,
|
||||
#[error("deploy/config.yaml was not found")]
|
||||
ConfigFileNotFound,
|
||||
#[error("configuration path is invalid")]
|
||||
InvalidConfigPath,
|
||||
#[error("configuration document is invalid: {path}")]
|
||||
InvalidDocument { path: PathBuf },
|
||||
#[error("配置文件 {path} 缺少必填字段 {field:?}")]
|
||||
MissingField { path: PathBuf, field: String },
|
||||
#[error("configuration field is invalid: {0}")]
|
||||
InvalidField(&'static str),
|
||||
#[error("environment variable is invalid: {0}")]
|
||||
InvalidEnvironment(&'static str),
|
||||
}
|
||||
|
||||
fn optional_environment_value(name: &'static str) -> Result<Option<String>, ConfigError> {
|
||||
match env::var(name) {
|
||||
Ok(value) if value.trim().is_empty() => Err(ConfigError::InvalidEnvironment(name)),
|
||||
|
||||
@@ -28,7 +28,34 @@ use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::lifecycle::ConnectionLifecycle;
|
||||
use gateway_core::task::{WorkerContribution, WorkerLeaderLeasePort};
|
||||
|
||||
pub use config::{ConfigError, HostConfig, LoadableConfig, load_config};
|
||||
pub use config::{HostConfig, LoadableConfig, load_config};
|
||||
|
||||
/// Host 配置加载与各能力配置校验共用的脱敏错误
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("配置文件 {path} 的字段 {field:?} 不合法")]
|
||||
InvalidValue {
|
||||
path: std::path::PathBuf,
|
||||
field: String,
|
||||
},
|
||||
#[error("current directory is unavailable")]
|
||||
CurrentDirectory,
|
||||
#[error("deploy/config.yaml was not found")]
|
||||
ConfigFileNotFound,
|
||||
#[error("configuration path is invalid")]
|
||||
InvalidConfigPath,
|
||||
#[error("configuration document is invalid: {path}")]
|
||||
InvalidDocument { path: std::path::PathBuf },
|
||||
#[error("配置文件 {path} 缺少必填字段 {field:?}")]
|
||||
MissingField {
|
||||
path: std::path::PathBuf,
|
||||
field: String,
|
||||
},
|
||||
#[error("configuration field is invalid: {0}")]
|
||||
InvalidField(&'static str),
|
||||
#[error("environment variable is invalid: {0}")]
|
||||
InvalidEnvironment(&'static str),
|
||||
}
|
||||
|
||||
use self::client_distribution::RgAdguardClientDistribution;
|
||||
use self::logging::{LogGuard, initialize_logging};
|
||||
|
||||
@@ -46,7 +46,7 @@ pub enum LogError {
|
||||
#[error("logging size limit is too large")]
|
||||
SizeOverflow,
|
||||
#[error(transparent)]
|
||||
InvalidConfig(#[from] crate::config::ConfigError),
|
||||
InvalidConfig(#[from] crate::ConfigError),
|
||||
}
|
||||
|
||||
/// 初始化日志;按各通道的保留窗口清理,不以分片数量淘汰记录
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::{
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{OperationError, conflict, internal};
|
||||
use crate::config::ConfigError;
|
||||
use crate::ConfigError;
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const DEFAULT_GITHUB_API_BASE: &str = "https://api.github.com/repos";
|
||||
|
||||
@@ -53,7 +53,7 @@ fn host_config_should_reject_empty_update_asset_directory() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
fn host_config_resolves_resource_paths_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
@@ -66,34 +66,10 @@ fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
config.logging.file.directory,
|
||||
PathBuf::from("/srv/gateway/.runtime/logs")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
config.runtime_data_dir(),
|
||||
PathBuf::from("/srv/gateway/runtime-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_derives_update_paths_from_runtime_data_dir() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
config.system_update.update_state_file,
|
||||
@@ -124,23 +100,19 @@ fn host_config_rejects_zero_worker_shutdown_window() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_rejects_zero_retention_windows() {
|
||||
for request_dump in [false, true] {
|
||||
let mut config = valid_config();
|
||||
if request_dump {
|
||||
config.logging.request_dump_retention_days = 0.0;
|
||||
} else {
|
||||
config.logging.file.retention_days = 0;
|
||||
}
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
fn file_logging_rejects_zero_retention() {
|
||||
let mut config = valid_config();
|
||||
config.logging.file.retention_days = 0;
|
||||
let error = config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_host::ConfigError::InvalidField("host.logging.file.retention_days")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,7 +12,7 @@ use gateway_admin::{
|
||||
instances::PluginInstance,
|
||||
state::{
|
||||
DeletePluginState, PluginStateConfiguration, PluginStateOwner,
|
||||
PluginStateOwnerRequest, PluginStateSchema, PutPluginState,
|
||||
PluginStateOwnerRequest, PutPluginState,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -25,7 +25,6 @@ use gateway_plugin_sdk::{
|
||||
StatePutRequest, StatePutResult, StateRecord,
|
||||
},
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::RpcReply;
|
||||
|
||||
@@ -43,7 +42,7 @@ impl PluginPrivateState {
|
||||
configuration: PluginStateConfiguration,
|
||||
owner: Option<PluginStateOwner>,
|
||||
) -> Result<Self, AdminError> {
|
||||
let validators = validators(manifest)?;
|
||||
let validators = crate::package::state_validators(manifest)?;
|
||||
Ok(Self {
|
||||
store,
|
||||
configuration,
|
||||
@@ -175,46 +174,6 @@ impl PluginPrivateState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn configuration(manifest: &Manifest) -> Result<PluginStateConfiguration, AdminError> {
|
||||
let _ = validators(manifest)?;
|
||||
let mut namespaces = Vec::with_capacity(manifest.state.len());
|
||||
for state in &manifest.state {
|
||||
let mut schema = state.schema.clone();
|
||||
schema.sort_all_objects();
|
||||
let encoded = serde_json::to_vec(&schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无法编码"))?;
|
||||
namespaces.push(PluginStateSchema {
|
||||
namespace: state.namespace.clone(),
|
||||
schema_version: state.schema_version,
|
||||
schema_sha256: hex::encode(Sha256::digest(encoded)),
|
||||
schema: state.schema.clone(),
|
||||
maximum_records: state.maximum_records,
|
||||
maximum_bytes: state.maximum_bytes,
|
||||
maximum_value_bytes: state.maximum_value_bytes,
|
||||
migrates_from: state.migrates_from.clone(),
|
||||
});
|
||||
}
|
||||
namespaces.sort_by(|left, right| left.namespace.cmp(&right.namespace));
|
||||
Ok(PluginStateConfiguration { namespaces })
|
||||
}
|
||||
|
||||
fn validators(manifest: &Manifest) -> Result<BTreeMap<String, jsonschema::Validator>, AdminError> {
|
||||
manifest
|
||||
.state
|
||||
.iter()
|
||||
.map(|state| {
|
||||
let validator = jsonschema::options()
|
||||
.offline()
|
||||
.with_pattern_options(
|
||||
jsonschema::PatternOptions::fancy_regex().backtrack_limit(20_000),
|
||||
)
|
||||
.build(&state.schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无效或引用了外部资源"))?;
|
||||
Ok((state.namespace.clone(), validator))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn map_store_fault(error: PluginStateStoreError) -> PluginFault {
|
||||
let (code, message) = match error.kind() {
|
||||
PluginStateStoreErrorKind::Invalid => (ErrorCode::InvalidInput, "state input is invalid"),
|
||||
|
||||
@@ -103,7 +103,7 @@ pub(super) fn validate(
|
||||
}
|
||||
Ok((
|
||||
configuration,
|
||||
crate::callback::private_state::configuration(manifest)?,
|
||||
crate::package::state_configuration(manifest)?,
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
//! 将已发布与准备中的实例事实投影为运行诊断,不参与候选提交
|
||||
|
||||
use super::super::restart_circuit::RestartIdentity;
|
||||
use super::restart_circuit::RestartIdentity;
|
||||
use super::{
|
||||
PluginRuntime,
|
||||
set::{PreparedSet, instance_fingerprint},
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
//! 准备单个插件实例的进程、RPC 会话与能力贡献
|
||||
|
||||
use super::super::restart_circuit::RestartIdentity;
|
||||
use super::restart_circuit::RestartIdentity;
|
||||
use super::{
|
||||
PluginRuntime,
|
||||
set::{PreparedContributions, PreparedInstance, instance_fingerprint},
|
||||
@@ -100,7 +100,7 @@ impl PluginRuntime {
|
||||
}
|
||||
let manifest = package.manifest();
|
||||
let (configuration, state_configuration) =
|
||||
super::super::configuration::validate(&instance, manifest, &schemas)?;
|
||||
super::configuration::validate(&instance, manifest, &schemas)?;
|
||||
crate::adapter::observer::validate_bindings(manifest, &instance.bindings)?;
|
||||
crate::adapter::policy::validate_bindings(manifest, &instance.bindings)?;
|
||||
crate::adapter::middleware::validate_bindings(manifest, &instance.bindings)?;
|
||||
@@ -139,7 +139,7 @@ impl PluginRuntime {
|
||||
.plugin_id()
|
||||
.map_err(|_| AdminError::invalid("插件身份无效"))?;
|
||||
let instance_id = instance.id.clone();
|
||||
crate::compatibility::report(&manifest, &instance_id)?;
|
||||
crate::deprecation::report(&manifest, &instance_id)?;
|
||||
let restart_identity =
|
||||
RestartIdentity::new(instance_id.clone(), authorization_binding.clone());
|
||||
let owner = self
|
||||
@@ -1,8 +1,15 @@
|
||||
//! 插件发布代次的配置校验、实例准备与重启熔断入口
|
||||
//! 插件发布代次的配置校验、实例生命周期与运行时端口入口
|
||||
|
||||
mod configuration;
|
||||
mod prepare;
|
||||
mod diagnostics;
|
||||
mod instance;
|
||||
mod maintenance;
|
||||
mod management;
|
||||
mod restart_circuit;
|
||||
mod runtime;
|
||||
mod set;
|
||||
mod snapshot;
|
||||
mod state;
|
||||
|
||||
pub use prepare::{PluginRuntime, PluginRuntimeConfig};
|
||||
pub use restart_circuit::PluginRestartCircuitConfig;
|
||||
pub use runtime::{PluginRuntime, PluginRuntimeConfig};
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
//! 插件运行时端口与代次准备的公共入口
|
||||
|
||||
mod diagnostics;
|
||||
mod instance;
|
||||
mod maintenance;
|
||||
mod management;
|
||||
mod runtime;
|
||||
mod set;
|
||||
mod snapshot;
|
||||
mod state;
|
||||
|
||||
pub use runtime::{PluginRuntime, PluginRuntimeConfig};
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
//! 插件运行时配置、端口装配与整体关闭;候选和调用均复用同一状态
|
||||
|
||||
use super::super::restart_circuit::{PluginRestartCircuitConfig, RestartCircuits, RestartIdentity};
|
||||
use super::restart_circuit::{PluginRestartCircuitConfig, RestartCircuits, RestartIdentity};
|
||||
use super::{
|
||||
diagnostics::PreparationDiagnostic,
|
||||
set::{PreparedInstance, PreparedSet},
|
||||
@@ -65,7 +65,7 @@ pub struct PluginRuntime {
|
||||
shutdown_lock: Mutex<()>,
|
||||
pub(super) processes: Arc<gateway_host::process::ProcessSupervisor>,
|
||||
pub(super) validators: Arc<Semaphore>,
|
||||
pub(super) configuration_schemas: Arc<super::super::configuration::ConfigurationSchemas>,
|
||||
pub(super) configuration_schemas: Arc<super::configuration::ConfigurationSchemas>,
|
||||
pub(super) log_slots: Arc<Semaphore>,
|
||||
pub(super) account_ports: Arc<PluginAccountPortSlot>,
|
||||
pub(super) client_key_ports: Arc<PluginClientKeyPortSlot>,
|
||||
+2
-7
@@ -363,12 +363,7 @@ impl PluginPreparation for PluginRuntime {
|
||||
let schemas = Arc::clone(&self.configuration_schemas);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _slot = slot;
|
||||
super::super::configuration::configuration_ready(
|
||||
&instance,
|
||||
&schema,
|
||||
&secret_fields,
|
||||
&schemas,
|
||||
)
|
||||
super::configuration::configuration_ready(&instance, &schema, &secret_fields, &schemas)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| AdminError::internal("插件校验任务失败"))?
|
||||
@@ -397,7 +392,7 @@ impl PluginPreparation for PluginRuntime {
|
||||
ValidatedPackage::read(artifact.archive, Some(&instance.artifact_sha256), limits)
|
||||
.map_err(|_| AdminError::invalid("插件制品校验失败"))?;
|
||||
let (_, state) =
|
||||
super::super::configuration::validate(&instance, package.manifest(), &schemas)?;
|
||||
super::configuration::validate(&instance, package.manifest(), &schemas)?;
|
||||
crate::adapter::observer::validate_bindings(package.manifest(), &instance.bindings)?;
|
||||
crate::adapter::policy::validate_bindings(package.manifest(), &instance.bindings)?;
|
||||
crate::adapter::middleware::validate_bindings(package.manifest(), &instance.bindings)?;
|
||||
@@ -2,17 +2,15 @@
|
||||
|
||||
mod adapter;
|
||||
mod callback;
|
||||
mod compatibility;
|
||||
mod contribution;
|
||||
mod deprecation;
|
||||
mod generation;
|
||||
mod package;
|
||||
mod rpc;
|
||||
mod stream;
|
||||
|
||||
pub use adapter::command_line::{PluginCommandError, PluginCommandOutput, PluginCommandSession};
|
||||
pub use generation::{PluginRestartCircuitConfig, PluginRuntime, PluginRuntimeConfig};
|
||||
pub use package::{
|
||||
PackageError, PackageInspector, PackageLimits, PreparedPackage, ValidatedPackage,
|
||||
};
|
||||
pub use rpc::{CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSession};
|
||||
pub use stream::RpcStream;
|
||||
pub use rpc::{CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSession, RpcStream};
|
||||
|
||||
@@ -101,8 +101,7 @@ impl PackageInspector {
|
||||
.collect::<Result<_, _>>()?,
|
||||
configuration_schema: manifest.configuration_schema.clone(),
|
||||
secret_fields: manifest.secret_fields.iter().cloned().collect(),
|
||||
state_namespaces: crate::callback::private_state::configuration(manifest)?
|
||||
.namespaces,
|
||||
state_namespaces: super::state_configuration(manifest)?.namespaces,
|
||||
};
|
||||
let requirements = super::compatibility::requirements(manifest)?;
|
||||
Ok((
|
||||
@@ -138,7 +137,7 @@ impl PluginPackageInspector for PackageInspector {
|
||||
metadata: &PluginArtifactMetadata,
|
||||
) -> Result<Vec<gateway_admin::model::plugins::instances::PluginApiDeprecation>, AdminError>
|
||||
{
|
||||
crate::compatibility::warnings(
|
||||
crate::deprecation::warnings(
|
||||
metadata
|
||||
.contributes
|
||||
.iter()
|
||||
|
||||
@@ -4,6 +4,7 @@ mod cache;
|
||||
mod compatibility;
|
||||
mod icon;
|
||||
mod inspection;
|
||||
mod state_schema;
|
||||
mod validation;
|
||||
|
||||
pub use cache::PreparedPackage;
|
||||
@@ -13,4 +14,7 @@ pub(crate) use compatibility::{
|
||||
};
|
||||
pub use inspection::PackageInspector;
|
||||
pub(crate) use inspection::inspection_error;
|
||||
pub(crate) use state_schema::{
|
||||
configuration as state_configuration, validators as state_validators,
|
||||
};
|
||||
pub use validation::{PackageError, PackageLimits, ValidatedPackage};
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
//! 将清单私有状态声明编译为配置、稳定摘要与离线校验器
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
plugins::state::{PluginStateConfiguration, PluginStateSchema},
|
||||
};
|
||||
use gateway_plugin_sdk::Manifest;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
pub(crate) fn configuration(manifest: &Manifest) -> Result<PluginStateConfiguration, AdminError> {
|
||||
let _ = validators(manifest)?;
|
||||
let mut namespaces = Vec::with_capacity(manifest.state.len());
|
||||
for state in &manifest.state {
|
||||
let mut schema = state.schema.clone();
|
||||
schema.sort_all_objects();
|
||||
let encoded = serde_json::to_vec(&schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无法编码"))?;
|
||||
namespaces.push(PluginStateSchema {
|
||||
namespace: state.namespace.clone(),
|
||||
schema_version: state.schema_version,
|
||||
schema_sha256: hex::encode(Sha256::digest(encoded)),
|
||||
schema: state.schema.clone(),
|
||||
maximum_records: state.maximum_records,
|
||||
maximum_bytes: state.maximum_bytes,
|
||||
maximum_value_bytes: state.maximum_value_bytes,
|
||||
migrates_from: state.migrates_from.clone(),
|
||||
});
|
||||
}
|
||||
namespaces.sort_by(|left, right| left.namespace.cmp(&right.namespace));
|
||||
Ok(PluginStateConfiguration { namespaces })
|
||||
}
|
||||
|
||||
pub(crate) fn validators(
|
||||
manifest: &Manifest,
|
||||
) -> Result<BTreeMap<String, jsonschema::Validator>, AdminError> {
|
||||
manifest
|
||||
.state
|
||||
.iter()
|
||||
.map(|state| {
|
||||
let validator = jsonschema::options()
|
||||
.offline()
|
||||
.with_pattern_options(
|
||||
jsonschema::PatternOptions::fancy_regex().backtrack_limit(20_000),
|
||||
)
|
||||
.build(&state.schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无效或引用了外部资源"))?;
|
||||
Ok((state.namespace.clone(), validator))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -18,8 +18,8 @@ use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}
|
||||
|
||||
use super::{
|
||||
CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSessionLoad, admission::ExecutionCapacity,
|
||||
stream::StreamIngress,
|
||||
};
|
||||
use crate::stream::StreamIngress;
|
||||
|
||||
pub(crate) struct Shared {
|
||||
state: Mutex<State>,
|
||||
|
||||
@@ -5,8 +5,10 @@ mod calls;
|
||||
mod contract;
|
||||
mod dispatch;
|
||||
mod session;
|
||||
mod stream;
|
||||
|
||||
pub(crate) use calls::Shared;
|
||||
pub use contract::{CallbackHandler, RpcError, RpcLimits, RpcReply};
|
||||
pub(crate) use contract::{RpcSessionDiagnostic, RpcSessionLifecycle, RpcSessionLoad};
|
||||
pub use session::RpcSession;
|
||||
pub use stream::RpcStream;
|
||||
|
||||
@@ -15,9 +15,9 @@ use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, mpsc};
|
||||
|
||||
use super::{
|
||||
CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSessionDiagnostic, RpcSessionLifecycle,
|
||||
RpcSessionLoad, Shared, dispatch,
|
||||
RpcSessionLoad, RpcStream, Shared, dispatch, stream::StreamIngress,
|
||||
};
|
||||
use crate::{PreparedPackage, RpcStream, stream::StreamIngress};
|
||||
use crate::PreparedPackage;
|
||||
|
||||
pub struct RpcSession {
|
||||
shared: Arc<Shared>,
|
||||
|
||||
+2
-3
@@ -6,11 +6,10 @@ use tokio::sync::{OwnedSemaphorePermit, mpsc, oneshot};
|
||||
|
||||
use gateway_plugin_sdk::client::CallDeadline;
|
||||
|
||||
use crate::{RpcError, RpcReply, RpcSession, rpc::Shared};
|
||||
|
||||
use super::super::{RpcError, RpcReply, RpcSession, Shared};
|
||||
use super::flow_control::ReceiveWindow;
|
||||
|
||||
pub(crate) struct StreamIngress {
|
||||
pub(in crate::rpc) struct StreamIngress {
|
||||
chunks: mpsc::Sender<Vec<u8>>,
|
||||
terminal: oneshot::Sender<Result<(), RpcError>>,
|
||||
window: ReceiveWindow,
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
//! 插件流的字节与帧数窗口校验、序号推进及信用归还
|
||||
|
||||
use crate::RpcError;
|
||||
use super::super::RpcError;
|
||||
|
||||
/// 字节与帧数一起授予,避免攻击者用大量单字节分块突破队列上限
|
||||
pub(super) struct ReceiveWindow {
|
||||
@@ -0,0 +1,7 @@
|
||||
//! RPC 会话内的响应流桥接与接收流量控制
|
||||
|
||||
mod bridge;
|
||||
mod flow_control;
|
||||
|
||||
pub use bridge::RpcStream;
|
||||
pub(super) use bridge::StreamIngress;
|
||||
@@ -1,7 +0,0 @@
|
||||
//! 插件 RPC 流桥接与接收流量控制的公共入口
|
||||
|
||||
mod bridge;
|
||||
mod flow_control;
|
||||
|
||||
pub use bridge::RpcStream;
|
||||
pub(crate) use bridge::StreamIngress;
|
||||
+8
-8
@@ -11,7 +11,7 @@ use gateway_core::routing::extensions::ExtensionPreparationPort;
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_distinguish_running_failed_preparation_and_draining_generations() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let snapshot = store.snapshot.lock().unwrap().clone();
|
||||
let active = PluginPreparation::prepare(&runtime, snapshot.clone())
|
||||
.await
|
||||
@@ -88,12 +88,12 @@ async fn diagnostics_distinguish_running_failed_preparation_and_draining_generat
|
||||
assert_eq!(draining.draining_revisions, [1]);
|
||||
|
||||
drop(active);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preparing_diagnostics_never_wait_for_the_serialized_prepare_io() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let source = store.snapshot.lock().unwrap().clone();
|
||||
let mut candidate = source.clone();
|
||||
candidate.config_revision = Revision::new(2).unwrap();
|
||||
@@ -123,12 +123,12 @@ async fn preparing_diagnostics_never_wait_for_the_serialized_prepare_io() {
|
||||
|
||||
let candidate = preparation.await.expect("delayed candidate");
|
||||
drop(candidate);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_report_a_published_process_fault_without_plugin_details() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
snapshot.instances[0].configuration = serde_json::json!({"exit_after_ready_ms":20});
|
||||
let active = PluginPreparation::prepare(&runtime, snapshot.clone())
|
||||
@@ -165,12 +165,12 @@ async fn diagnostics_report_a_published_process_fault_without_plugin_details() {
|
||||
);
|
||||
|
||||
drop(active);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_published_process_crash_preserves_the_serving_snapshot_and_other_plugin_status() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let snapshot = {
|
||||
let mut snapshot = store.snapshot.lock().unwrap();
|
||||
let mut failed = snapshot.instances[0].clone();
|
||||
@@ -202,5 +202,5 @@ async fn a_published_process_crash_preserves_the_serving_snapshot_and_other_plug
|
||||
PluginInstanceRuntimeStatus::Faulted
|
||||
);
|
||||
drop(generation);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
+29
-30
@@ -21,7 +21,7 @@ fn startup_count(marker: &Path) -> usize {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cold_preparation_overlaps_startup_with_bounded_concurrency() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
let template = snapshot.instances[0].clone();
|
||||
let gate = cache.path().join("startup-gate");
|
||||
@@ -65,21 +65,20 @@ async fn cold_preparation_overlaps_startup_with_bounded_concurrency() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_with_a_changed_contribution_id_is_rejected() {
|
||||
let (cache, store, runtime) = super::super::setup_with_contributions(Contributions::from([
|
||||
crate::support::contribution(
|
||||
let (cache, store, runtime) =
|
||||
super::setup_with_contributions(Contributions::from([crate::support::contribution(
|
||||
Capability::Observer,
|
||||
vec![Stage::Observation],
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
),
|
||||
]))
|
||||
.await;
|
||||
)]))
|
||||
.await;
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
let candidate = PluginPreparation::prepare(&runtime, snapshot.clone())
|
||||
.await
|
||||
.expect("the declared contribution prepares before its registration id changes");
|
||||
drop(candidate);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
|
||||
snapshot.instances[0].configuration = serde_json::json!({"registration_mismatch":true});
|
||||
let error = PluginPreparation::prepare(&runtime, snapshot)
|
||||
@@ -97,7 +96,7 @@ async fn restart_circuit_caps_short_lived_crashes_and_a_new_revision_recovers()
|
||||
maximum_failures: NonZeroUsize::new(3).unwrap(),
|
||||
stability_window: Duration::from_secs(1),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("starts.jsonl");
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
@@ -165,7 +164,7 @@ async fn restart_circuit_caps_short_lived_crashes_and_a_new_revision_recovers()
|
||||
drop(active);
|
||||
drop(disabled_generation);
|
||||
drop(replacement);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -174,7 +173,7 @@ async fn restart_circuit_resets_after_a_stable_incarnation() {
|
||||
maximum_failures: NonZeroUsize::new(3).unwrap(),
|
||||
stability_window: Duration::from_millis(80),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("stable-reset-starts.jsonl");
|
||||
let exit = controls.path().join("stable-exit");
|
||||
@@ -217,7 +216,7 @@ async fn restart_circuit_resets_after_a_stable_incarnation() {
|
||||
assert_eq!(startup_count(&marker), 5);
|
||||
|
||||
drop(active);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -226,7 +225,7 @@ async fn restart_circuit_ignores_planned_generation_shutdown() {
|
||||
maximum_failures: NonZeroUsize::MIN,
|
||||
stability_window: Duration::from_secs(60),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("planned-shutdown-starts.jsonl");
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
@@ -252,7 +251,7 @@ async fn restart_circuit_ignores_planned_generation_shutdown() {
|
||||
|
||||
drop(first);
|
||||
drop(second);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -261,7 +260,7 @@ async fn restart_circuit_counts_exit_before_the_candidate_is_indexed() {
|
||||
maximum_failures: NonZeroUsize::new(3).unwrap(),
|
||||
stability_window: Duration::from_secs(1),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("registration-exits.jsonl");
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
@@ -300,7 +299,7 @@ async fn restart_circuit_counts_exit_before_the_candidate_is_indexed() {
|
||||
.map(|failure| failure.code.as_str()),
|
||||
Some("restart_circuit_open")
|
||||
);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -309,7 +308,7 @@ async fn restart_circuit_counts_a_started_process_that_exits_during_handshake()
|
||||
maximum_failures: NonZeroUsize::new(2).unwrap(),
|
||||
stability_window: Duration::from_millis(10),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("handshake-exits.jsonl");
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
@@ -336,7 +335,7 @@ async fn restart_circuit_counts_a_started_process_that_exits_during_handshake()
|
||||
2,
|
||||
"host-side gating must happen before another process spawn"
|
||||
);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -345,7 +344,7 @@ async fn reused_session_failure_counts_once_across_multiple_generations() {
|
||||
maximum_failures: NonZeroUsize::new(3).unwrap(),
|
||||
stability_window: Duration::from_millis(50),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("shared-generations.jsonl");
|
||||
let exit = controls.path().join("shared-exit");
|
||||
@@ -419,7 +418,7 @@ async fn reused_session_failure_counts_once_across_multiple_generations() {
|
||||
drop(retained);
|
||||
drop(recovered);
|
||||
drop(published);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -428,7 +427,7 @@ async fn abandoned_candidate_keeps_shared_process_failure_in_restart_budget() {
|
||||
maximum_failures: NonZeroUsize::MIN,
|
||||
stability_window: Duration::from_secs(1),
|
||||
};
|
||||
let (cache, store, runtime) = super::super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
|
||||
let controls = tempfile::tempdir().unwrap();
|
||||
let marker = controls.path().join("abandoned-candidate.jsonl");
|
||||
let old_exit = controls.path().join("abandoned-candidate-old-exit");
|
||||
@@ -508,7 +507,7 @@ async fn abandoned_candidate_keeps_shared_process_failure_in_restart_budget() {
|
||||
|
||||
drop(recovered);
|
||||
drop(published);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -527,7 +526,7 @@ async fn incompatible_start_requires_scoped_confirmation_and_rechecks_after_rest
|
||||
contribution.1.version = 3;
|
||||
}
|
||||
let (cache, store, initial) =
|
||||
super::super::setup_with_contributions(Contributions::from([contribution])).await;
|
||||
super::setup_with_contributions(Contributions::from([contribution])).await;
|
||||
drop(initial);
|
||||
let host = if mismatch == "host" { "3.0.0" } else { "1.0.0" };
|
||||
let runtime = compatibility_runtime(&cache, store.clone(), host);
|
||||
@@ -624,7 +623,7 @@ async fn incompatible_start_requires_scoped_confirmation_and_rechecks_after_rest
|
||||
drop(confirmed);
|
||||
drop(restored);
|
||||
runtime.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
|
||||
let restarted = compatibility_runtime(&cache, store.clone(), host);
|
||||
let restored =
|
||||
@@ -645,7 +644,7 @@ async fn incompatible_start_requires_scoped_confirmation_and_rechecks_after_rest
|
||||
assert_eq!(startup_count(&other_marker), 0);
|
||||
drop(restored);
|
||||
restarted.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -681,7 +680,7 @@ async fn unloadable_packages_have_no_guards_and_cannot_be_forced_to_start() {
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
for defect in ["rpc", "manifest", "platform", "archive", "digest"] {
|
||||
let (cache, store, initial) = super::super::setup().await;
|
||||
let (cache, store, initial) = super::setup().await;
|
||||
drop(initial);
|
||||
let mut store = Arc::try_unwrap(store).ok().unwrap();
|
||||
let mut broken = store.artifacts.values().next().unwrap().clone();
|
||||
@@ -808,13 +807,13 @@ async fn unloadable_packages_have_no_guards_and_cannot_be_forced_to_start() {
|
||||
assert!(restored.is_ready());
|
||||
drop(restored);
|
||||
runtime.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn declared_registration_budget_is_not_cut_short_by_process_preparation_timeout() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
snapshot.instances[0].configuration = serde_json::json!({
|
||||
"registration_delay_ms":31_000,
|
||||
@@ -829,12 +828,12 @@ async fn declared_registration_budget_is_not_cut_short_by_process_preparation_ti
|
||||
.expect("ready process may register beyond the startup control deadline");
|
||||
assert!(candidate.is_ready());
|
||||
drop(candidate);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cancels_long_registration_and_reclaims_the_unpublished_candidate() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let (cache, store, runtime) = super::setup().await;
|
||||
let runtime = std::sync::Arc::new(runtime);
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let started = markers.path().join("registration");
|
||||
@@ -861,5 +860,5 @@ async fn shutdown_cancels_long_registration_and_reclaims_the_unpublished_candida
|
||||
"shutdown must cancel candidate preparation instead of waiting for its long budget",
|
||||
);
|
||||
assert!(preparing.await.unwrap().is_err());
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
@@ -1,23 +1,26 @@
|
||||
//! 验证插件维护任务的资源创建、重试及事务归属和幂等性
|
||||
//! 验证维护任务的跨发布保活、取消顺序、资源创建重试及事务归属和幂等性
|
||||
|
||||
use crate::support::environment::{Environment, mutation};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
Revision,
|
||||
account_groups::{AccountGroupColor, CreateAccountGroup},
|
||||
plugin_resources::{GroupMembersChange, ManagedKeyConfig, PluginResourceOwner},
|
||||
plugins::instances::PluginInstanceRuntimeStatus,
|
||||
plugins::instances::{PluginInstanceRuntimeStatus, PluginInstanceSnapshot},
|
||||
},
|
||||
ports::plugin_resources::PluginResourceAccess,
|
||||
ports::{plugin_resources::PluginResourceAccess, plugins::PluginPreparation},
|
||||
};
|
||||
use gateway_core::task::WorkerDaemonContext;
|
||||
use gateway_core::{
|
||||
account::FastMode,
|
||||
lifecycle::CancellationToken,
|
||||
task::{WorkerContribution, WorkerRunnable},
|
||||
routing::{ConfigRevision, RuntimeSnapshot},
|
||||
runtime::RuntimeSnapshotHandle,
|
||||
task::{WorkerContribution, WorkerDaemonContext, WorkerRunnable},
|
||||
};
|
||||
use gateway_plugin_runtime::PluginRuntime;
|
||||
use gateway_plugin_sdk::{Capability, Contributions, Stage};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
fn group(name: &str) -> CreateAccountGroup {
|
||||
CreateAccountGroup {
|
||||
@@ -201,6 +204,50 @@ async fn wait_done(path: &std::path::Path, after: usize, phase: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
async fn publish(
|
||||
runtime: &PluginRuntime,
|
||||
snapshots: &RuntimeSnapshotHandle,
|
||||
instances: PluginInstanceSnapshot,
|
||||
) {
|
||||
let revision = ConfigRevision::new(instances.config_revision.get()).unwrap();
|
||||
let prepared = PluginPreparation::prepare(runtime, instances)
|
||||
.await
|
||||
.unwrap();
|
||||
snapshots.publish(
|
||||
RuntimeSnapshot::new(
|
||||
revision,
|
||||
gateway_core::settings::SettingsValues::new(1, "smart", Default::default(), None, None),
|
||||
vec![],
|
||||
vec![],
|
||||
vec![],
|
||||
)
|
||||
.unwrap()
|
||||
.with_extensions(Some(prepared)),
|
||||
);
|
||||
}
|
||||
|
||||
fn records(path: &Path) -> Vec<Value> {
|
||||
std::fs::read_to_string(path)
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str(line).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn wait_records(path: &Path, count: usize) -> Vec<Value> {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let entries = records(path);
|
||||
if entries.len() >= count {
|
||||
return entries;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("missing maintenance records: {error}; {:?}", records(path)))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unused_maintenance_response_does_not_stop_the_host_worker() {
|
||||
let Some(environment) = Environment::create().await else {
|
||||
@@ -509,3 +556,99 @@ async fn resource_transactions_enforce_ownership_current_revision_and_idempotent
|
||||
drop(runtime);
|
||||
environment.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrelated_publications_preserve_inflight_maintenance_and_replacement_waits_for_cancel() {
|
||||
let (cache, store, runtime) =
|
||||
super::setup_with_contributions(Contributions::from([crate::support::contribution(
|
||||
Capability::Maintenance,
|
||||
vec![Stage::Maintenance],
|
||||
vec![],
|
||||
vec![],
|
||||
)]))
|
||||
.await;
|
||||
let runtime = Arc::new(runtime);
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let mut instances = store.snapshot.lock().unwrap().clone();
|
||||
let template = instances.instances[0].clone();
|
||||
instances.instances = ["a", "b"]
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let mut instance = template.clone();
|
||||
instance.id = id.into();
|
||||
instance.configuration = json!({
|
||||
"maintenance_marker":markers.path().join(id),
|
||||
"execution_cancelled_marker":markers.path().join(id),
|
||||
"maintenance_wait_marker":markers.path().join(format!("release-{id}")),
|
||||
"call_context_marker":markers.path().join(format!("context-{id}")),
|
||||
"call_budgets_ms":{"maintenance":600_000},
|
||||
});
|
||||
instance
|
||||
})
|
||||
.collect();
|
||||
let snapshots = RuntimeSnapshotHandle::default();
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let WorkerContribution::Registration(registration) =
|
||||
runtime.maintenance_worker(snapshots.clone()).unwrap()
|
||||
else {
|
||||
panic!("maintenance worker registration")
|
||||
};
|
||||
let WorkerRunnable::Daemon { task, .. } = registration.runnable else {
|
||||
panic!("maintenance daemon")
|
||||
};
|
||||
let stop = CancellationToken::new();
|
||||
let running = tokio::spawn({
|
||||
let stop = stop.clone();
|
||||
async move { task.run(WorkerDaemonContext::new(stop)).await.unwrap() }
|
||||
});
|
||||
let a = markers.path().join("a");
|
||||
let b = markers.path().join("b");
|
||||
assert_eq!(wait_records(&a, 1).await[0]["phase"], "start");
|
||||
assert_eq!(wait_records(&b, 1).await[0]["phase"], "start");
|
||||
let contexts = records(&markers.path().join("context-a"));
|
||||
let maintenance = contexts
|
||||
.iter()
|
||||
.find(|entry| entry["method"] == "plugin.reconcile")
|
||||
.expect("maintenance call context");
|
||||
assert_eq!(maintenance["context"]["timeout_ms"], 600_000);
|
||||
|
||||
instances.config_revision = Revision::new(2).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let changed = wait_records(&b, 3).await;
|
||||
assert!(
|
||||
changed[1].get("call_id").is_some(),
|
||||
"旧维护必须先完成取消确认"
|
||||
);
|
||||
assert_eq!(changed[2]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "无关实例发布不得重启 A 的在途维护");
|
||||
|
||||
instances.config_revision = Revision::new(3).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = false;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert!(wait_records(&b, 4).await[3].get("call_id").is_some());
|
||||
instances.config_revision = Revision::new(4).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = true;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert_eq!(wait_records(&b, 5).await[4]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "停用与恢复 B 不得取消 A");
|
||||
|
||||
// 多次发布通知合并为一次后续对账,首轮仍使用原调用 ID 完成
|
||||
std::fs::write(markers.path().join("release-a"), []).unwrap();
|
||||
let completed = wait_records(&a, 4).await;
|
||||
assert_eq!(completed[1]["phase"], "done");
|
||||
assert_eq!(completed[1]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed[2]["phase"], "start");
|
||||
assert_eq!(completed[3]["phase"], "done");
|
||||
assert_ne!(completed[2]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed.len(), 4);
|
||||
|
||||
stop.cancel();
|
||||
running.await.unwrap();
|
||||
assert!(wait_records(&b, 6).await[5].get("call_id").is_some());
|
||||
snapshots.suspend();
|
||||
runtime.shutdown().await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
+4
-5
@@ -9,15 +9,14 @@ use gateway_plugin_sdk::Stage;
|
||||
|
||||
#[tokio::test]
|
||||
async fn management_needs_no_binding_and_rejects_stale_execution_identity_configuration() {
|
||||
let (_cache, store, runtime) = super::super::setup_with_contributions(Contributions::from([
|
||||
crate::support::contribution(
|
||||
let (_cache, store, runtime) =
|
||||
super::setup_with_contributions(Contributions::from([crate::support::contribution(
|
||||
Capability::Management,
|
||||
vec![Stage::Management],
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
),
|
||||
]))
|
||||
.await;
|
||||
)]))
|
||||
.await;
|
||||
let mut instance = store.snapshot.lock().unwrap().instances[0].clone();
|
||||
assert!(
|
||||
PluginPreparation::validate(&runtime, instance.clone())
|
||||
@@ -3,11 +3,16 @@
|
||||
mod authentication;
|
||||
mod configuration;
|
||||
mod data;
|
||||
mod diagnostics;
|
||||
mod instance;
|
||||
mod maintenance;
|
||||
mod management;
|
||||
mod observer;
|
||||
mod policy;
|
||||
mod prepare;
|
||||
mod private_state;
|
||||
mod runtime;
|
||||
mod set;
|
||||
mod snapshot;
|
||||
mod upstream_adapter;
|
||||
|
||||
use gateway_admin::{
|
||||
@@ -25,6 +30,7 @@ use gateway_plugin_sdk::Contributions;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use crate::support::store::Store;
|
||||
@@ -114,3 +120,13 @@ async fn wait_until_empty(cache: &std::path::Path) {
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn wait_until_unready(reference: &gateway_core::routing::extensions::ExtensionSetReference) {
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while reference.is_ready() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker exit observed");
|
||||
}
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
//! 验证维护任务跨发布保活、通知合并及实例替换后的取消顺序
|
||||
|
||||
use gateway_admin::{
|
||||
model::{Revision, plugins::instances::PluginInstanceSnapshot},
|
||||
ports::plugins::PluginPreparation,
|
||||
};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
routing::{ConfigRevision, RuntimeSnapshot},
|
||||
runtime::RuntimeSnapshotHandle,
|
||||
task::{WorkerContribution, WorkerDaemonContext, WorkerRunnable},
|
||||
};
|
||||
use gateway_plugin_runtime::PluginRuntime;
|
||||
use gateway_plugin_sdk::{Capability, Contributions, Stage};
|
||||
use serde_json::{Value, json};
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
async fn publish(
|
||||
runtime: &PluginRuntime,
|
||||
snapshots: &RuntimeSnapshotHandle,
|
||||
instances: PluginInstanceSnapshot,
|
||||
) {
|
||||
let revision = ConfigRevision::new(instances.config_revision.get()).unwrap();
|
||||
let prepared = PluginPreparation::prepare(runtime, instances)
|
||||
.await
|
||||
.unwrap();
|
||||
snapshots.publish(
|
||||
RuntimeSnapshot::new(
|
||||
revision,
|
||||
gateway_core::settings::SettingsValues::new(1, "smart", Default::default(), None, None),
|
||||
vec![],
|
||||
vec![],
|
||||
vec![],
|
||||
)
|
||||
.unwrap()
|
||||
.with_extensions(Some(prepared)),
|
||||
);
|
||||
}
|
||||
|
||||
fn records(path: &Path) -> Vec<Value> {
|
||||
std::fs::read_to_string(path)
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str(line).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn wait_records(path: &Path, count: usize) -> Vec<Value> {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let entries = records(path);
|
||||
if entries.len() >= count {
|
||||
return entries;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("missing maintenance records: {error}; {:?}", records(path)))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrelated_publications_preserve_inflight_maintenance_and_replacement_waits_for_cancel() {
|
||||
let (cache, store, runtime) = super::super::setup_with_contributions(Contributions::from([
|
||||
crate::support::contribution(
|
||||
Capability::Maintenance,
|
||||
vec![Stage::Maintenance],
|
||||
vec![],
|
||||
vec![],
|
||||
),
|
||||
]))
|
||||
.await;
|
||||
let runtime = Arc::new(runtime);
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let mut instances = store.snapshot.lock().unwrap().clone();
|
||||
let template = instances.instances[0].clone();
|
||||
instances.instances = ["a", "b"]
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let mut instance = template.clone();
|
||||
instance.id = id.into();
|
||||
instance.configuration = json!({
|
||||
"maintenance_marker":markers.path().join(id),
|
||||
"execution_cancelled_marker":markers.path().join(id),
|
||||
"maintenance_wait_marker":markers.path().join(format!("release-{id}")),
|
||||
"call_context_marker":markers.path().join(format!("context-{id}")),
|
||||
"call_budgets_ms":{"maintenance":600_000},
|
||||
});
|
||||
instance
|
||||
})
|
||||
.collect();
|
||||
let snapshots = RuntimeSnapshotHandle::default();
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let WorkerContribution::Registration(registration) =
|
||||
runtime.maintenance_worker(snapshots.clone()).unwrap()
|
||||
else {
|
||||
panic!("maintenance worker registration")
|
||||
};
|
||||
let WorkerRunnable::Daemon { task, .. } = registration.runnable else {
|
||||
panic!("maintenance daemon")
|
||||
};
|
||||
let stop = CancellationToken::new();
|
||||
let running = tokio::spawn({
|
||||
let stop = stop.clone();
|
||||
async move { task.run(WorkerDaemonContext::new(stop)).await.unwrap() }
|
||||
});
|
||||
let a = markers.path().join("a");
|
||||
let b = markers.path().join("b");
|
||||
assert_eq!(wait_records(&a, 1).await[0]["phase"], "start");
|
||||
assert_eq!(wait_records(&b, 1).await[0]["phase"], "start");
|
||||
let contexts = records(&markers.path().join("context-a"));
|
||||
let maintenance = contexts
|
||||
.iter()
|
||||
.find(|entry| entry["method"] == "plugin.reconcile")
|
||||
.expect("maintenance call context");
|
||||
assert_eq!(maintenance["context"]["timeout_ms"], 600_000);
|
||||
|
||||
instances.config_revision = Revision::new(2).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let changed = wait_records(&b, 3).await;
|
||||
assert!(
|
||||
changed[1].get("call_id").is_some(),
|
||||
"旧维护必须先完成取消确认"
|
||||
);
|
||||
assert_eq!(changed[2]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "无关实例发布不得重启 A 的在途维护");
|
||||
|
||||
instances.config_revision = Revision::new(3).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = false;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert!(wait_records(&b, 4).await[3].get("call_id").is_some());
|
||||
instances.config_revision = Revision::new(4).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = true;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert_eq!(wait_records(&b, 5).await[4]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "停用与恢复 B 不得取消 A");
|
||||
|
||||
// 多次发布通知合并为一次后续对账,首轮仍使用原调用 ID 完成
|
||||
std::fs::write(markers.path().join("release-a"), []).unwrap();
|
||||
let completed = wait_records(&a, 4).await;
|
||||
assert_eq!(completed[1]["phase"], "done");
|
||||
assert_eq!(completed[1]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed[2]["phase"], "start");
|
||||
assert_eq!(completed[3]["phase"], "done");
|
||||
assert_ne!(completed[2]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed.len(), 4);
|
||||
|
||||
stop.cancel();
|
||||
running.await.unwrap();
|
||||
assert!(wait_records(&b, 6).await[5].get("call_id").is_some());
|
||||
snapshots.suspend();
|
||||
runtime.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
//! 插件发布准备各职责的集成测试入口
|
||||
|
||||
mod diagnostics;
|
||||
mod instance;
|
||||
mod maintenance;
|
||||
mod management;
|
||||
mod runtime;
|
||||
mod set;
|
||||
mod snapshot;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
async fn wait_until_unready(reference: &gateway_core::routing::extensions::ExtensionSetReference) {
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while reference.is_ready() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker exit observed");
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user