mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
96e8443ede | ||
|
|
42badf64ca | ||
|
|
35afe1123f | ||
|
|
2ac4265832 | ||
|
|
d22ec29824 | ||
|
|
7a32c6cec2 | ||
|
|
0ff1a72919 | ||
|
|
79083a5632 | ||
|
|
90f7456db8 | ||
|
|
6f71b27c3c | ||
|
|
7b05b14a2b | ||
|
|
f9b5b5ff77 | ||
|
|
4d5778a84f | ||
|
|
6d22d73506 | ||
|
|
d8ab207282 | ||
|
|
3816b46ba1 | ||
|
|
4a7c06f895 | ||
|
|
e0b7a841ca | ||
|
|
96a292dd0f | ||
|
|
6f138c8256 | ||
|
|
c8ffb1a8c8 | ||
|
|
79df82a4f0 | ||
|
|
62516d9769 | ||
|
|
892cba68ae | ||
|
|
fada9bc0bd | ||
|
|
27b9bbe994 | ||
|
|
db849c1251 | ||
|
|
c57af9b0dd |
@@ -7,21 +7,18 @@ RUN --mount=type=bind,target=/cache \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& cp -a /cache/registry/. /usr/local/cargo/registry/ \
|
||||
&& cp -a /cache/git/. /usr/local/cargo/git/ \
|
||||
&& cp -a /cache/target/. /app/backend/target/
|
||||
&& tar -xzf /cache/registry.tar.gz -C /usr/local/cargo/registry \
|
||||
&& tar -xzf /cache/git.tar.gz -C /usr/local/cargo/git \
|
||||
&& tar -xzf /cache/target.tar.gz -C /app/backend/target
|
||||
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS collect
|
||||
FROM busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092 AS snapshot
|
||||
ARG CACHE_STAMP
|
||||
# 一次导出三个 mount,避免逐目录创建镜像、载入 Docker 再复制文件。
|
||||
# 直接压缩 mount 内容,构建层和 runner 只保存归档,避免多份展开的编译产物占满磁盘
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/backend/target \
|
||||
test -n "$CACHE_STAMP" \
|
||||
&& mkdir -p /cache/registry /cache/git /cache/target \
|
||||
&& cp -a /usr/local/cargo/registry/. /cache/registry/ \
|
||||
&& cp -a /usr/local/cargo/git/. /cache/git/ \
|
||||
&& cp -a /app/backend/target/. /cache/target/
|
||||
|
||||
FROM scratch AS snapshot
|
||||
COPY --from=collect /cache/ /
|
||||
&& mkdir -p /cache \
|
||||
&& tar -czf /cache/registry.tar.gz -C /usr/local/cargo/registry . \
|
||||
&& tar -czf /cache/git.tar.gz -C /usr/local/cargo/git . \
|
||||
&& tar -czf /cache/target.tar.gz -C /app/backend/target .
|
||||
|
||||
@@ -14,24 +14,42 @@ case "$architecture" in amd64|arm64) ;; *) exit 2 ;; esac
|
||||
dockerfile="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Dockerfile"
|
||||
|
||||
usable() {
|
||||
local root="$1" path crates=false libraries=false
|
||||
# 目录或 buildstamp 存在不代表编译发生过,必须同时有下载包与目标依赖。
|
||||
for path in "$root"/registry/cache/*/*.crate; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then crates=true; break; fi
|
||||
done
|
||||
for path in "$root"/target/"$architecture"/release/deps/*.rlib; do
|
||||
if [ -f "$path" ] && [ -s "$path" ]; then libraries=true; break; fi
|
||||
done
|
||||
[ "$crates" = true ] && [ "$libraries" = true ]
|
||||
# 流式检查归档中的非空依赖文件,校验时不在 runner 展开编译产物
|
||||
python3 - "$1" "$architecture" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
architecture = sys.argv[2]
|
||||
|
||||
def contains(filename, pattern):
|
||||
with tarfile.open(root / filename, mode="r|gz") as archive:
|
||||
return any(
|
||||
member.isfile() and member.size > 0
|
||||
and re.fullmatch(pattern, member.name.removeprefix("./"))
|
||||
for member in archive
|
||||
)
|
||||
|
||||
try:
|
||||
valid = (
|
||||
(root / "git.tar.gz").stat().st_size > 0
|
||||
and contains("registry.tar.gz", r"cache/[^/]+/[^/]+\.crate")
|
||||
and contains("target.tar.gz", rf"{architecture}/release/deps/[^/]+\.rlib")
|
||||
)
|
||||
except (OSError, tarfile.TarError):
|
||||
valid = False
|
||||
sys.exit(0 if valid else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
command=(docker buildx build --builder "$builder" --file "$dockerfile" --progress plain)
|
||||
# mount 的内容不参与层指纹,用时间戳重跑复制;--no-cache 会重置 cache mount。
|
||||
# mount 的内容不参与层指纹,用时间戳重跑归档操作;--no-cache 会重置 cache mount
|
||||
command+=(--build-arg "CACHE_STAMP=$(date +%s%N)" --target "$operation")
|
||||
valid=false
|
||||
if [ "$operation" = restore ]; then
|
||||
if usable "$directory"; then
|
||||
mkdir -p "$directory"/{registry,git,target}
|
||||
"${command[@]}" --output type=cacheonly "$directory"
|
||||
valid=true
|
||||
else
|
||||
@@ -43,10 +61,11 @@ else
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,只接收 /cache 下的压缩归档
|
||||
# 直接导出打包阶段,避免再创建一层归档副本
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
| tar -xf - -C "$temporary/snapshot" --strip-components=1 cache/
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
@@ -36,10 +36,9 @@ jobs:
|
||||
with:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
# 同一平台和依赖复用快照,源码变化由 Cargo 指纹判定是否重编译
|
||||
key: container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
key: container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml', 'deploy/Dockerfile', '.github/actions/cargo-cache/Dockerfile') }}
|
||||
restore-keys: |
|
||||
container-cargo-v3-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
container-cargo-v4-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
- name: Restore Cargo mounts to BuildKit
|
||||
id: cargo-restore
|
||||
@@ -50,6 +49,9 @@ jobs:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
architecture: amd64
|
||||
|
||||
- name: Release restored Cargo snapshot
|
||||
run: rm -rf "$RUNNER_TEMP/container-cargo"
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail
|
||||
--test-threads 8 --no-tests fail
|
||||
|
||||
- name: Store tests with Redis ACL authentication
|
||||
working-directory: backend
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
cargo nextest run \
|
||||
--cargo-metadata "$RUNNER_TEMP/nextest-cargo-metadata.json" \
|
||||
--binaries-metadata "$RUNNER_TEMP/nextest-binaries.json" \
|
||||
--no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
--test-threads 8 --no-tests fail -E 'package(=gateway-store) & test(/^redis::/)'
|
||||
|
||||
frontend:
|
||||
if: inputs.run_frontend
|
||||
|
||||
@@ -171,6 +171,12 @@ jobs:
|
||||
toolchain: ${{ needs.release-info.outputs.rust }}
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Configure Windows runtime linkage
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
# 发行包静态链接 MSVC 运行库,解压运行时无需另外安装 VC++ Redistributable
|
||||
run: echo 'RUSTFLAGS=-C target-feature=+crt-static' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
@@ -186,6 +192,8 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cargo build --release --locked --target "${{ matrix.target }}" --bin codex-proxy-rs
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --help
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --version
|
||||
mkdir -p "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}"
|
||||
cp "target/${{ matrix.target }}/release/${{ matrix.binary }}" "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/${{ matrix.binary }}"
|
||||
working-directory: backend
|
||||
@@ -195,6 +203,16 @@ jobs:
|
||||
CPR_BUILD_TIME: ${{ needs.release-info.outputs.build_time }}
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Verify Windows platform behavior
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
working-directory: backend
|
||||
env:
|
||||
RUST_MIN_STACK: '16777216'
|
||||
run: |
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-admin --test main use_case::plugins::official::
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-host --test main system_update::
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
@@ -40,6 +40,8 @@
|
||||
|
||||
以下命令适用于 Linux amd64/arm64,需要 Docker Engine、Docker Compose Plugin、curl 和 OpenSSL
|
||||
|
||||
Windows x64 / ARM64 使用发行包部署,步骤见 [Windows 二进制部署](deploy/README.md#windows-二进制部署)
|
||||
|
||||
已有部署请先看 [升级说明](deploy/README.md#镜像升级与源码构建),不要覆盖原配置
|
||||
|
||||
### 一键安装
|
||||
|
||||
Generated
+40
-38
@@ -286,9 +286,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.155.0"
|
||||
version = "1.156.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a542f0e9261b9ce56708d591a62062079fa3c58a362ffbf49a2e48a4f6ced16"
|
||||
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.3"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -387,9 +387,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.64.2"
|
||||
version = "0.64.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398"
|
||||
checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -408,9 +408,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.5.1"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e"
|
||||
checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -432,9 +432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666"
|
||||
checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -452,9 +452,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -475,9 +475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.1"
|
||||
version = "1.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -492,9 +492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
|
||||
checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -514,9 +514,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -537,9 +537,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.64.0"
|
||||
version = "0.64.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413"
|
||||
checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -2485,9 +2485,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.0"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e7305938cac5ee144352ceb71583ffc6cfeac7d20fcb20ef865a4d5c314e499"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2513,18 +2513,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8a848331614144c03e815fb931f238c9e342bd7ab501b29e35d647e7cd5688"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5a947ef798eeacacb43edf29b1d0ab64bdf5e2c9fbcaad10ee8663d922b544"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -3578,9 +3578,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.1"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -3634,9 +3634,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "922268812b6f9e20b4ab0f1efe8e07036958eed7346ceddb31c777b5531fd11f"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3759,9 +3759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.21.1"
|
||||
version = "0.21.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
|
||||
checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -4953,9 +4953,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
@@ -5052,9 +5052,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.28.0"
|
||||
version = "1.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5646,16 +5646,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -49,6 +49,10 @@ opt-level = 2
|
||||
[profile.dev.package."miniz_oxide@0.9.1"]
|
||||
opt-level = 2
|
||||
|
||||
# workspace 的 WebSocket deflate 特性会让 flate2 选用 zlib-rs
|
||||
[profile.dev.package.zlib-rs]
|
||||
opt-level = 2
|
||||
|
||||
[profile.release]
|
||||
debug = false
|
||||
strip = "symbols"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! 网关命令行入口,选择启动模式并创建异步运行时
|
||||
|
||||
use std::{
|
||||
env,
|
||||
error::Error,
|
||||
io::{self, Write},
|
||||
process::ExitCode,
|
||||
@@ -21,7 +20,7 @@ fn main() -> ExitCode {
|
||||
|
||||
fn run() -> Result<ExitCode, Box<dyn Error + Send + Sync>> {
|
||||
ensure_rustls_provider();
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
let mut arguments = gateway_host::system_update::startup_arguments()?;
|
||||
let command = arguments
|
||||
.next()
|
||||
.map(|value| {
|
||||
|
||||
@@ -70,6 +70,7 @@ fn gateway_core_depends_on_no_http_db_redis_or_provider_crate() {
|
||||
#[test]
|
||||
fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let allowed: Option<&[&str]> = match relative.to_str().expect("source path") {
|
||||
"validation.rs" => Some(&[]),
|
||||
@@ -96,7 +97,7 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
let Some(allowed) = allowed else { continue };
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
let syntax = syn::parse_file(&source).expect("parse core source");
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
// 相对路径中的 self 指向当前 owner,不是跨 owner 依赖
|
||||
references.owners.remove(&references.module_path[0]);
|
||||
@@ -111,17 +112,24 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_owner_paths_form_a_dag() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
fn domain_owner_paths_form_a_dag() {
|
||||
for member in ["crates/gateway-core", "crates/gateway-admin"] {
|
||||
assert_owner_paths_form_a_dag(member);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_owner_paths_form_a_dag(member: &str) {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
let mut graph = BTreeMap::<String, BTreeSet<String>>::new();
|
||||
for relative in super::rust_files(&root) {
|
||||
if relative == Path::new("lib.rs") {
|
||||
continue;
|
||||
}
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let mut references = CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
let owner = references.module_path[0].clone();
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse core source"));
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
references.owners.remove(&owner);
|
||||
graph.entry(owner).or_default().extend(references.owners);
|
||||
}
|
||||
@@ -131,7 +139,7 @@ fn core_owner_paths_form_a_dag() {
|
||||
while let Some(dependency) = pending.pop() {
|
||||
assert_ne!(
|
||||
&dependency, owner,
|
||||
"Core owner dependency cycle reaches {owner}"
|
||||
"{member} owner dependency cycle reaches {owner}"
|
||||
);
|
||||
if visited.insert(dependency.clone())
|
||||
&& let Some(next) = graph.get(&dependency)
|
||||
@@ -143,7 +151,44 @@ fn core_owner_paths_form_a_dag() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
fn shared_owners_do_not_depend_on_their_callers() {
|
||||
let boundaries: &[(&str, &str, &[&str])] = &[
|
||||
("crates/providers/xai", "transport", &["credential"]),
|
||||
("crates/providers/openai", "transport", &["provider"]),
|
||||
("crates/gateway-api", "time", &["admin"]),
|
||||
(
|
||||
"crates/gateway-api",
|
||||
"client_error",
|
||||
&["admin", "openai", "middleware"],
|
||||
),
|
||||
("crates/gateway-api", "middleware", &["openai"]),
|
||||
("crates/gateway-host", "system_update", &["config"]),
|
||||
("crates/gateway-plugin/runtime", "package", &["callback"]),
|
||||
];
|
||||
for &(member, owner, forbidden) in boundaries {
|
||||
let root = backend_root().join(member).join("src");
|
||||
let reexports = read_root_reexport_owners(&root);
|
||||
for relative in super::rust_files(&root) {
|
||||
let mut references =
|
||||
CrateReferences::in_module(&relative).with_root_reexports(&reexports);
|
||||
if references.module_path.first().map(String::as_str) != Some(owner) {
|
||||
continue;
|
||||
}
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read owner source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse owner source"));
|
||||
for dependency in forbidden {
|
||||
assert!(
|
||||
!references.owners.contains(*dependency),
|
||||
"{member}/src/{} must not depend on crate::{dependency}",
|
||||
relative.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
pub use crate::{engine::AttemptContext, settings::SettingsValues as Values};
|
||||
@@ -167,13 +212,86 @@ fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
#[test]
|
||||
fn owner_reference_scan_resolves_crate_root_reexports() {
|
||||
let root = syn::parse_file(
|
||||
r#"
|
||||
mod provider;
|
||||
mod transport;
|
||||
mod secret;
|
||||
pub use provider::{OFFICIAL_URL, Options as ProviderOptions};
|
||||
pub use self::transport::{self as network, Request};
|
||||
pub use crate::secret::SecretValue;
|
||||
pub use external::Unrelated;
|
||||
"#,
|
||||
)
|
||||
.expect("parse root reexport fixture");
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
use crate::{OFFICIAL_URL, ProviderOptions, network::Client};
|
||||
use super::super::SecretValue;
|
||||
fn request() -> crate::Request { unimplemented!() }
|
||||
"#,
|
||||
)
|
||||
.expect("parse reexport consumer fixture");
|
||||
let reexports = root_reexport_owners(&root);
|
||||
assert!(!reexports.contains_key("Unrelated"));
|
||||
let mut references = CrateReferences::in_module(Path::new("consumer/request.rs"))
|
||||
.with_root_reexports(&reexports);
|
||||
references.visit_file(&syntax);
|
||||
assert_eq!(
|
||||
references.owners,
|
||||
BTreeSet::from([
|
||||
"provider".to_owned(),
|
||||
"transport".to_owned(),
|
||||
"secret".to_owned(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences {
|
||||
fn read_root_reexport_owners(root: &Path) -> BTreeMap<String, String> {
|
||||
let source = fs::read_to_string(root.join("lib.rs")).expect("read crate root");
|
||||
root_reexport_owners(&syn::parse_file(&source).expect("parse crate root"))
|
||||
}
|
||||
|
||||
// 只解析根模块的显式重导出,不推断 glob、宏展开或完整 Rust 名称解析
|
||||
fn root_reexport_owners(syntax: &syn::File) -> BTreeMap<String, String> {
|
||||
let modules = syntax
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| match item {
|
||||
Item::Mod(module) => Some(module.ident.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut reexports = BTreeMap::new();
|
||||
for item in &syntax.items {
|
||||
if let Item::Use(item) = item
|
||||
&& !matches!(item.vis, syn::Visibility::Inherited)
|
||||
{
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, name| {
|
||||
let owner = path
|
||||
.iter()
|
||||
.find(|segment| !matches!(segment.as_str(), "crate" | "self"));
|
||||
if let (Some(owner), Some(name)) = (owner, name)
|
||||
&& modules.contains(owner)
|
||||
{
|
||||
reexports.insert(name.to_owned(), owner.clone());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
reexports
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences<'a> {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
root_reexports: Option<&'a BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences<'_> {
|
||||
fn visit_path(&mut self, path: &'ast syn::Path) {
|
||||
self.record(
|
||||
&path
|
||||
@@ -186,11 +304,13 @@ impl<'ast> Visit<'ast> for CrateReferences {
|
||||
}
|
||||
|
||||
fn visit_item_use(&mut self, item: &'ast syn::ItemUse) {
|
||||
self.use_tree(&item.tree, &mut Vec::new());
|
||||
visit_use_leaves(&item.tree, &mut Vec::new(), &mut |path, _| {
|
||||
self.record(path);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl CrateReferences {
|
||||
impl<'a> CrateReferences<'a> {
|
||||
fn in_module(relative: &Path) -> Self {
|
||||
let mut module = relative
|
||||
.with_extension("")
|
||||
@@ -208,13 +328,26 @@ impl CrateReferences {
|
||||
Self {
|
||||
owners: BTreeSet::new(),
|
||||
module_path: module,
|
||||
root_reexports: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_root_reexports(mut self, reexports: &'a BTreeMap<String, String>) -> Self {
|
||||
self.root_reexports = Some(reexports);
|
||||
self
|
||||
}
|
||||
|
||||
fn root_owner(&self, name: &str) -> String {
|
||||
self.root_reexports
|
||||
.and_then(|reexports| reexports.get(name))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| name.to_owned())
|
||||
}
|
||||
|
||||
fn record(&mut self, path: &[String]) {
|
||||
let Some(first) = path.first() else { return };
|
||||
let owner = if first == "crate" {
|
||||
path.get(1).cloned()
|
||||
path.get(1).map(|name| self.root_owner(name))
|
||||
} else if first == "self" || first == "super" {
|
||||
let mut module = self.module_path.clone();
|
||||
let mut rest = path;
|
||||
@@ -227,7 +360,10 @@ impl CrateReferences {
|
||||
}
|
||||
rest = &rest[1..];
|
||||
}
|
||||
module.first().or_else(|| rest.first()).cloned()
|
||||
module
|
||||
.first()
|
||||
.cloned()
|
||||
.or_else(|| rest.first().map(|name| self.root_owner(name)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -235,31 +371,41 @@ impl CrateReferences {
|
||||
self.owners.insert(owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn use_tree(&mut self, tree: &syn::UseTree, prefix: &mut Vec<String>) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
self.use_tree(&path.tree, prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
prefix.push(name.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
self.use_tree(item, prefix);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => self.record(prefix),
|
||||
fn visit_use_leaves(
|
||||
tree: &syn::UseTree,
|
||||
prefix: &mut Vec<String>,
|
||||
visit: &mut impl FnMut(&[String], Option<&str>),
|
||||
) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
visit_use_leaves(&path.tree, prefix, visit);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) => {
|
||||
let name = name.ident.to_string();
|
||||
let export = if name == "self" {
|
||||
prefix.last().cloned()
|
||||
} else {
|
||||
Some(name.clone())
|
||||
};
|
||||
prefix.push(name);
|
||||
visit(prefix, export.as_deref());
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
visit(prefix, Some(&rename.rename.to_string()));
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
visit_use_leaves(item, prefix, visit);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) => visit(prefix, None),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,11 +11,15 @@ use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerDaemonContext, WorkerTaskError},
|
||||
task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId,
|
||||
WorkerKind, WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStatusTransition,
|
||||
BackupStorageConfig, BackupTriggerKind, build_backup_seed, code,
|
||||
@@ -34,6 +38,8 @@ impl From<BackupError> for WorkerTaskError {
|
||||
|
||||
use super::policy::{BackupSchedule, decide_retention};
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
|
||||
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量
|
||||
@@ -72,6 +78,22 @@ impl BackupTask {
|
||||
self.timezone = timezone;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
//! 管理服务与后台任务的启动组装,窄端口供服务器和 CLI 复用
|
||||
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{engine::probe::AccountProbe, runtime::SnapshotControl};
|
||||
|
||||
use crate::{
|
||||
AccountsService, AdminBundle, AdminConfig, AdminRuntimePorts, AdminServices, ClientConfig,
|
||||
ClientKeyService, CredentialsService, PluginDistributionPorts, PluginManagementService,
|
||||
PluginsService, SettingsService, UsageDataService, backup, freeze_recovery,
|
||||
model::AdminError,
|
||||
ports::{
|
||||
self, plugin_accounts::PluginAccountAccess, plugin_client_keys::PluginClientKeyAccess,
|
||||
provider::ProviderAdminRegistry, store::AdminStorePorts,
|
||||
},
|
||||
public_service, reset_credit_worker, use_case,
|
||||
use_case::{
|
||||
account_groups::DefaultAccountGroupService,
|
||||
accounts::DefaultAccountsService,
|
||||
auth::{AuthService, DefaultAuthService},
|
||||
backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService,
|
||||
client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService,
|
||||
settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
},
|
||||
};
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
providers.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(
|
||||
use_case::system_preflight::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
),
|
||||
);
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
providers.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), providers.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
providers.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
providers.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
providers.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let worker_contributions = vec![
|
||||
backup_task.into_worker()?,
|
||||
import_task.into_worker()?,
|
||||
freeze_recovery.into_worker()?,
|
||||
reset_credit_worker::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}
|
||||
.into_worker()?,
|
||||
];
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
@@ -8,9 +8,13 @@ use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountErrorProtection, ProviderAccountId};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::AdminError;
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
@@ -42,6 +46,34 @@ impl FreezeRecoveryTask {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
FREEZE_RECOVERY_INTERVAL,
|
||||
WORKER_INITIAL_BACKOFF,
|
||||
WORKER_MAXIMUM_BACKOFF,
|
||||
WORKER_LEASE_TTL,
|
||||
WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
|
||||
/// 无法读取或校验配置时跳过本轮,不能把未知状态当作允许解冻
|
||||
async fn freeze_policy(&self) -> Option<AccountErrorProtection> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
|
||||
@@ -2,26 +2,22 @@
|
||||
//!
|
||||
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现
|
||||
|
||||
use std::{fmt, path::Path, sync::Arc, time::Duration};
|
||||
use std::{fmt, path::Path, sync::Arc};
|
||||
|
||||
use gateway_core::{
|
||||
engine::execution::ClientKeyVerifier,
|
||||
engine::probe::AccountProbe,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
engine::execution::ClientKeyVerifier, engine::probe::AccountProbe, runtime::SnapshotControl,
|
||||
task::WorkerContribution,
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
mod bootstrap;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
pub mod public_service;
|
||||
mod reset_credit_automation;
|
||||
mod reset_credit_worker;
|
||||
mod use_case;
|
||||
pub use use_case::plugins::{PluginDistributionPorts, PluginManagementService, PluginsService};
|
||||
|
||||
@@ -44,18 +40,15 @@ pub use use_case::{
|
||||
usage_data::UsageDataService,
|
||||
};
|
||||
|
||||
use model::AdminError;
|
||||
pub use bootstrap::{
|
||||
initialize, initialize_accounts, initialize_plugin_accounts, initialize_plugin_client_keys,
|
||||
initialize_plugin_resources, initialize_settings, initialize_usage_data,
|
||||
initialize_with_plugin_accounts,
|
||||
};
|
||||
|
||||
use ports::{
|
||||
client_distribution::ClientDistributionResolver, plugin_accounts::PluginAccountAccess,
|
||||
plugin_client_keys::PluginClientKeyAccess, provider::ProviderAdminRegistry,
|
||||
store::AdminStorePorts, system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
account_groups::DefaultAccountGroupService, accounts::DefaultAccountsService,
|
||||
auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_distribution::DefaultClientDistributionService, client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService, settings::DefaultSettingsService,
|
||||
system::DefaultSystemService,
|
||||
provider::ProviderAdminRegistry, system::SystemOperations,
|
||||
};
|
||||
|
||||
const MINIMUM_INITIAL_PASSWORD_BYTES: usize = 12;
|
||||
@@ -70,7 +63,6 @@ const WEAK_ADMIN_PASSWORDS: &[&str] = &[
|
||||
"codex-proxy-rs",
|
||||
];
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
const DEFAULT_CLIENT_SESSION_TTL_MINUTES: u64 = 24 * 60;
|
||||
|
||||
/// 只用于首次幂等创建默认管理员的启动密码
|
||||
@@ -366,395 +358,3 @@ pub struct AdminRuntimePorts {
|
||||
pub system: Arc<dyn SystemOperations>,
|
||||
pub client_key_verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
/// 校验配置、接入已组装的 Provider 注册表并完成默认管理员幂等初始化
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 配置非法或默认管理员初始化失败时返回错误
|
||||
pub async fn initialize(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, None).await
|
||||
}
|
||||
|
||||
/// 使用组合根已绑定给 Runtime 的同一账号窄端口,避免为完整 Admin 重建第二实例
|
||||
pub async fn initialize_with_plugin_accounts(
|
||||
config: AdminConfig,
|
||||
client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Arc<dyn PluginAccountAccess>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
initialize_inner(config, client_config, store, runtime, Some(plugin_accounts)).await
|
||||
}
|
||||
|
||||
async fn initialize_inner(
|
||||
mut config: AdminConfig,
|
||||
mut client_config: ClientConfig,
|
||||
store: AdminStorePorts,
|
||||
runtime: AdminRuntimePorts,
|
||||
plugin_accounts: Option<Arc<dyn PluginAccountAccess>>,
|
||||
) -> Result<AdminBundle, AdminError> {
|
||||
let AdminRuntimePorts {
|
||||
upstream_info,
|
||||
timezone,
|
||||
service_middleware,
|
||||
plugin_preparation,
|
||||
plugin_management,
|
||||
published_snapshot,
|
||||
plugin_distribution,
|
||||
plugin_inspector,
|
||||
pricing_source,
|
||||
providers,
|
||||
snapshot,
|
||||
account_probe: probe,
|
||||
proxy_probe,
|
||||
client_distribution,
|
||||
system,
|
||||
client_key_verifier,
|
||||
} = runtime;
|
||||
config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
client_config
|
||||
.resolve_and_validate(Path::new("."))
|
||||
.map_err(|error| AdminError::invalid(error.to_string()))?;
|
||||
let registry = providers;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
));
|
||||
auth.ensure_default_admin(config.default_password.expose())
|
||||
.await?;
|
||||
|
||||
let accounts = initialize_accounts(
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
store.settings(),
|
||||
timezone,
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
upstream_info.clone(),
|
||||
);
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
timezone,
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
)
|
||||
.with_timezone(timezone);
|
||||
let system_preflight = Arc::new(use_case::plugin_update::PluginSystemUpdatePreflight::new(
|
||||
store.plugins(),
|
||||
plugin_inspector.clone(),
|
||||
));
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
let credentials = Arc::new(CredentialsService::new(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
store.proxies(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
));
|
||||
let plugin_accounts = plugin_accounts.unwrap_or_else(|| {
|
||||
initialize_plugin_accounts(
|
||||
registry.clone(),
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
upstream_info.clone(),
|
||||
)
|
||||
});
|
||||
let import_tasks = use_case::import_tasks::DefaultImportTasksService::new(credentials.clone());
|
||||
let import_task = use_case::import_tasks::ImportTaskWorker(import_tasks.clone());
|
||||
let settings = initialize_settings(
|
||||
store.settings(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), registry.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_accounts(&accounts)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
peer_link: Arc::new(use_case::peer_link::DefaultPeerLinkService::new(
|
||||
store.client_keys(),
|
||||
store.accounts(),
|
||||
store.account_runtime(),
|
||||
registry.clone(),
|
||||
)),
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
plugin_management: Arc::new(PluginManagementService::new(
|
||||
plugin_management,
|
||||
store.plugins(),
|
||||
published_snapshot.clone(),
|
||||
)),
|
||||
plugins: Arc::new(PluginsService::new(
|
||||
store.plugins(),
|
||||
plugin_inspector,
|
||||
PluginDistributionPorts::new(plugin_distribution, store.proxies()),
|
||||
snapshot.clone(),
|
||||
plugin_preparation,
|
||||
published_snapshot,
|
||||
store.plugin_state(),
|
||||
)),
|
||||
key_usage,
|
||||
proxies: Arc::new(use_case::proxies::DefaultProxiesService::new(
|
||||
store.proxies(),
|
||||
proxy_probe,
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
upstream_info.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
registry.clone(),
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
upstream_info,
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(import_task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(reset_credit_automation::ResetCreditAutomationTask {
|
||||
accounts,
|
||||
store: store.accounts(),
|
||||
settings: store.settings(),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?,
|
||||
));
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// CLI 只组合账号用例,不初始化管理员、管理服务或后台任务;写入仍复用同一事务与审计
|
||||
pub fn initialize_plugin_accounts(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
snapshot: Arc<dyn gateway_core::runtime::SnapshotControl>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn PluginAccountAccess> {
|
||||
Arc::new(use_case::plugin_accounts::DefaultPluginAccountAccess::new(
|
||||
providers,
|
||||
accounts,
|
||||
snapshot,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 为 Runtime 创建非秘密 Client Key 目录与预算重置的窄端口
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_client_keys(
|
||||
providers: ports::provider::ProviderAdminRegistry,
|
||||
store: Arc<dyn ports::store::ClientKeyStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn PluginClientKeyAccess> {
|
||||
let service: Arc<dyn ClientKeyService> =
|
||||
Arc::new(DefaultClientKeyService::new(store, snapshot, providers));
|
||||
Arc::new(use_case::plugin_client_keys::DefaultPluginClientKeyAccess::new(service))
|
||||
}
|
||||
|
||||
/// 为 Runtime 组合实例自有资源写入;权限和归属在同一存储事务复核
|
||||
#[must_use]
|
||||
pub fn initialize_plugin_resources(
|
||||
store: Arc<dyn ports::plugin_resources::PluginResourceStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Arc<dyn ports::plugin_resources::PluginResourceAccess> {
|
||||
Arc::new(use_case::plugin_resources::DefaultPluginResourceAccess { store, snapshot })
|
||||
}
|
||||
|
||||
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`
|
||||
fn backup_worker_contribution(
|
||||
task: backup::task::BackupTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("备份 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("备份 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("备份 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 服务端与 CLI 复用账号用例,不随命令启动后台调度
|
||||
#[must_use]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "账号存储、发布、探测与互联分别通过独立端口注入"
|
||||
)]
|
||||
pub fn initialize_accounts(
|
||||
accounts: Arc<dyn ports::store::AccountStore>,
|
||||
account_runtime: Arc<dyn ports::store::AccountRuntimeStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
upstream_info: Option<Arc<dyn ports::peer::PeerUpstreamService>>,
|
||||
) -> Arc<dyn AccountsService> {
|
||||
Arc::new(DefaultAccountsService::new(
|
||||
accounts,
|
||||
account_runtime,
|
||||
settings,
|
||||
timezone,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
upstream_info,
|
||||
))
|
||||
}
|
||||
|
||||
/// 设置服务不依赖 Web 管理会话,CLI 与服务器通过同一用例执行事务和发布
|
||||
pub fn initialize_settings(
|
||||
store: Arc<dyn ports::store::SettingsStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
providers: ProviderAdminRegistry,
|
||||
pricing_source: Arc<dyn ports::pricing::PricingSource>,
|
||||
) -> Arc<dyn SettingsService> {
|
||||
Arc::new(DefaultSettingsService::new(
|
||||
store,
|
||||
snapshot,
|
||||
providers,
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -739,6 +739,8 @@ pub struct UsageListRecord {
|
||||
/// 请求选号时冻结的账号套餐
|
||||
pub provider_account_plan_type: Option<String>,
|
||||
pub provider_account_plan_type_display: Option<String>,
|
||||
/// 查询时的互联识别结果,缺少当前摘要时为未知
|
||||
pub provider_account_upstream_peer: Option<bool>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
@@ -1439,6 +1441,8 @@ pub struct DiagnosticsItem {
|
||||
pub name: String,
|
||||
pub account_authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
/// 仅账号维度补充当前互联识别结果
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
|
||||
@@ -165,8 +165,9 @@ pub struct ConfigurePluginInstance {
|
||||
pub artifact_sha256: String,
|
||||
pub enabled: bool,
|
||||
pub configuration: serde_json::Value,
|
||||
/// 缺省保留旧值;显式空对象清除全部 secret
|
||||
pub secrets: Option<BTreeMap<String, SecretString>>,
|
||||
/// 按字段更新,省略的字段保留,null 清除对应字段
|
||||
#[serde(default)]
|
||||
pub secrets: BTreeMap<String, Option<SecretString>>,
|
||||
pub bindings: Vec<PluginCapabilityBinding>,
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
//! 每日错峰发现与已知卡片的临期执行,周期只读取本地到期计划
|
||||
|
||||
use crate::{
|
||||
model::reset_credit_automation::{PlannedResetCredit, ResetCreditAutomation},
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{ScheduledTask, WorkerCycleContext, WorkerTaskError},
|
||||
time::DeploymentTimeZone,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
pub(crate) fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
pub(crate) fn consume_at(
|
||||
account_id: &str,
|
||||
credit: &PlannedResetCredit,
|
||||
lead_minutes: u32,
|
||||
) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
pub(crate) fn next_consumption(
|
||||
record: &ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
pub(crate) fn reschedule(
|
||||
record: &mut ResetCreditAutomation,
|
||||
lead_minutes: u32,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//! 自动重置卡 Worker 的到期扫描、并发预算与取消,消费编排由账号用例负责
|
||||
|
||||
use crate::{
|
||||
model::AdminError,
|
||||
ports::store::{AccountStore, SettingsStore},
|
||||
use_case::accounts::AccountsService,
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId, WorkerKind,
|
||||
WorkerLeaseRequest, WorkerRegistration, WorkerRunnable, WorkerSchedule, WorkerTaskError,
|
||||
},
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
pub(crate) struct ResetCreditAutomationTask {
|
||||
pub(crate) accounts: Arc<dyn AccountsService>,
|
||||
pub(crate) store: Arc<dyn AccountStore>,
|
||||
pub(crate) settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
impl ResetCreditAutomationTask {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::ResetCreditAutomation, "admin")
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(90),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), Duration::from_secs(90))
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("重置卡 Worker 注册失败"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for ResetCreditAutomationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡设置读取失败"))?;
|
||||
// 单轮有界,积压沿到期顺序继续;并发复用已有账号刷新配置
|
||||
let due = self
|
||||
.store
|
||||
.due_reset_credit_automations(Utc::now(), 100)
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡计划读取失败"))?;
|
||||
let work = futures::stream::iter(due)
|
||||
.map(|account_id| async move {
|
||||
let id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| WorkerTaskError::safe("重置卡账号 ID 不合法"))?;
|
||||
if let Err(error) = self.accounts.reconcile_reset_credit_automation(id).await {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error_kind = error.kind().as_str(),
|
||||
"自动重置卡任务失败"
|
||||
);
|
||||
}
|
||||
Ok::<_, WorkerTaskError>(())
|
||||
})
|
||||
.buffer_unordered(settings.values.refresh_concurrency.max(1) as usize)
|
||||
.collect::<Vec<_>>();
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
_ = tokio::time::timeout(Duration::from_secs(30), work) => {},
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,13 @@
|
||||
//! 重置卡自动使用的账号锁、持久幂等键、查询缓存与重试编排
|
||||
//! 重置卡查询、消费与自动使用排程,统一账号锁、持久幂等键和重试编排
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::model::{MutationActor, reset_credit_automation::*};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::time::DeploymentTimeZone;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
pub(super) async fn reset_credits_upstream(
|
||||
&self,
|
||||
@@ -51,13 +57,6 @@ impl DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
use crate::{
|
||||
model::{MutationActor, reset_credit_automation::*},
|
||||
reset_credit_automation::{consume_at, next_consumption, next_discovery, reschedule},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
|
||||
impl DefaultAccountsService {
|
||||
async fn automation_settings(
|
||||
&self,
|
||||
@@ -511,3 +510,54 @@ impl DefaultAccountsService {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 当地次日随机发现时间;随机数由调用者提供,结果持久化后不会因重启重抽
|
||||
fn next_discovery(
|
||||
now: DateTime<Utc>,
|
||||
timezone: DeploymentTimeZone,
|
||||
random: u64,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
let start = timezone.days_after(now, 1)?;
|
||||
let end = timezone.days_after(now, 2)?;
|
||||
let seconds = u64::try_from((end - start).num_seconds()).ok()?;
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
start.checked_add_signed(chrono::Duration::seconds(
|
||||
i64::try_from(random % seconds).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// 提前窗口的前 10% 用于账号错峰,最多一分钟,其余时间保留给网络失败恢复
|
||||
fn consume_at(account_id: &str, credit: &PlannedResetCredit, lead_minutes: u32) -> DateTime<Utc> {
|
||||
let lead_seconds = i64::from(lead_minutes) * 60;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(account_id.as_bytes());
|
||||
hash.update([0]);
|
||||
hash.update(credit.id.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let offset =
|
||||
i64::from(u16::from_be_bytes([digest[0], digest[1]])) % (lead_seconds / 10).clamp(1, 60);
|
||||
credit.expires_at - chrono::Duration::seconds(lead_seconds - offset)
|
||||
}
|
||||
|
||||
fn next_consumption(record: &ResetCreditAutomation, lead_minutes: u32) -> Option<DateTime<Utc>> {
|
||||
record
|
||||
.state
|
||||
.credits
|
||||
.iter()
|
||||
.map(|credit| consume_at(&record.account_id, credit, lead_minutes))
|
||||
.min()
|
||||
}
|
||||
|
||||
fn reschedule(record: &mut ResetCreditAutomation, lead_minutes: u32, now: DateTime<Utc>) {
|
||||
record.next_run_at = if let Some(retry) = record.state.retry_at {
|
||||
retry
|
||||
} else if record.state.pending.is_some() {
|
||||
now
|
||||
} else {
|
||||
next_consumption(record, lead_minutes).map_or(record.state.next_discovery_at, |due| {
|
||||
due.min(record.state.next_discovery_at)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -9,7 +9,10 @@ use std::{
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::task::{DaemonTask, WorkerDaemonContext, WorkerTaskError};
|
||||
use gateway_core::task::{
|
||||
DaemonRestartPolicy, DaemonTask, WorkerContribution, WorkerDaemonContext, WorkerId, WorkerKind,
|
||||
WorkerRegistration, WorkerRunnable, WorkerTaskError,
|
||||
};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -328,6 +331,24 @@ impl ImportTasksService for DefaultImportTasksService {
|
||||
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl ImportTaskWorker {
|
||||
pub(crate) fn into_worker(self) -> Result<WorkerContribution, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("导入 Worker 重启策略不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(self),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
Ok(WorkerContribution::Registration(registration))
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
|
||||
@@ -14,11 +14,11 @@ pub mod peer_link;
|
||||
pub mod plugin_accounts;
|
||||
pub mod plugin_client_keys;
|
||||
pub mod plugin_resources;
|
||||
pub(crate) mod plugin_update;
|
||||
pub mod plugins;
|
||||
pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub(crate) mod system_preflight;
|
||||
pub mod usage_data;
|
||||
|
||||
mod credential_mutation;
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
|
||||
use crate::{
|
||||
@@ -133,7 +134,8 @@ impl DefaultObservabilityService {
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
let runtime_slots = observation.runtime_slots;
|
||||
self.enrich_list_records(&mut observation.recent_requests);
|
||||
self.enrich_list_records(&mut observation.recent_requests)
|
||||
.await;
|
||||
self.enrich_dashboard_quotas(&mut observation.account_usage)
|
||||
.await;
|
||||
for account in &mut observation.account_usage {
|
||||
@@ -259,7 +261,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.list_usage_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage records"))?;
|
||||
self.enrich_list_records(&mut page.items);
|
||||
self.enrich_list_records(&mut page.items).await;
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -336,6 +338,12 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.usage_diagnostics(range, filter, dimension, 100)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage diagnostics"))?;
|
||||
let peers = if dimension == DiagnosticDimension::Account {
|
||||
self.current_upstream_peers(observation.items.iter().map(|item| item.key.as_str()))
|
||||
.await
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let items = observation
|
||||
.items
|
||||
.into_iter()
|
||||
@@ -345,6 +353,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
rate_or_zero(item.non_completion_count, item.request_count);
|
||||
let retry_rate = rate_or_zero(item.retried_request_count, item.request_count);
|
||||
DiagnosticsItem {
|
||||
account_upstream_peer: peers.get(&item.key).copied(),
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
account_authentication_kind: item.account_authentication_kind,
|
||||
@@ -912,6 +921,29 @@ fn usd_cost(costs: &[CurrencyCost]) -> Option<DecimalAmount> {
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
async fn current_upstream_peers<'a>(
|
||||
&self,
|
||||
ids: impl Iterator<Item = &'a str>,
|
||||
) -> HashMap<String, bool> {
|
||||
let Some(peer) = self.upstream_info.as_deref() else {
|
||||
return HashMap::new();
|
||||
};
|
||||
let ids = ids
|
||||
.map(str::to_owned)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
// 同一页重复账号只读取一次摘要,单个账号不存在或读取失败不阻断历史记录
|
||||
futures::stream::iter(ids)
|
||||
.map(|id: String| async move {
|
||||
let account_id = ProviderAccountId::new(id.clone()).ok()?;
|
||||
let upstream = peer.read_summary(&account_id).await.ok()??.upstream?;
|
||||
Some((id, upstream.discovered && upstream.observed_at.is_some()))
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.filter_map(|value| async move { value })
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn enrich_dashboard_quotas(&self, accounts: &mut [DashboardAccountUsage]) {
|
||||
if accounts.is_empty() {
|
||||
return;
|
||||
@@ -976,8 +1008,22 @@ impl DefaultObservabilityService {
|
||||
|
||||
/// 补充当前套餐展示名称,逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回
|
||||
fn enrich_list_records(&self, records: &mut [crate::model::observability::UsageListRecord]) {
|
||||
async fn enrich_list_records(
|
||||
&self,
|
||||
records: &mut [crate::model::observability::UsageListRecord],
|
||||
) {
|
||||
let peers = self
|
||||
.current_upstream_peers(
|
||||
records
|
||||
.iter()
|
||||
.filter_map(|record| record.provider_account_ref.as_deref()),
|
||||
)
|
||||
.await;
|
||||
for record in records {
|
||||
record.provider_account_upstream_peer = record
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.and_then(|id| peers.get(id).copied());
|
||||
record.provider_account_plan_type_display =
|
||||
record.provider_kind.as_deref().and_then(|provider| {
|
||||
self.providers
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
peer_link::{PeerLinkExport, PeerLinkLimits},
|
||||
provider_credentials::ProviderQuotaReadRequest,
|
||||
provider_credentials::{ProviderQuotaReadRequest, explicit_plan_type},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
@@ -175,10 +175,16 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "peer account runtime"))?;
|
||||
let mut quotas = BTreeMap::new();
|
||||
if sharing.share_quota {
|
||||
if sharing.share_quota || sharing.share_plans {
|
||||
let mut window_count = 0usize;
|
||||
let mut by_provider = BTreeMap::new();
|
||||
for account in &accounts {
|
||||
// 仅公开套餐时只补读缺失套餐的本地额度,不回源刷新或扩大公开范围
|
||||
if !sharing.share_quota
|
||||
&& explicit_plan_type(account.plan_type.as_deref()).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
by_provider
|
||||
.entry(&account.provider_kind)
|
||||
.or_insert_with(Vec::new)
|
||||
@@ -200,9 +206,11 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.zip(provider.read_quotas(&requests).await)
|
||||
{
|
||||
if let Ok(quota) = quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
if sharing.share_quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
}
|
||||
}
|
||||
quotas.insert(account.id.clone(), quota);
|
||||
}
|
||||
@@ -215,6 +223,7 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
if sharing.share_pool {
|
||||
result.pool = Some(projection::pool_summary(
|
||||
&accounts,
|
||||
"as,
|
||||
&runtime,
|
||||
&self.providers,
|
||||
sharing.share_plans,
|
||||
|
||||
@@ -25,6 +25,7 @@ use crate::{
|
||||
|
||||
pub(super) fn pool_summary(
|
||||
accounts: &[AccountRecord],
|
||||
quotas: &BTreeMap<String, ProviderQuota>,
|
||||
runtime: &AccountRuntimeSnapshot,
|
||||
providers: &ProviderAdminRegistry,
|
||||
share_plans: bool,
|
||||
@@ -43,12 +44,13 @@ pub(super) fn pool_summary(
|
||||
AccountStatus::Error => summary.error += 1,
|
||||
}
|
||||
if let Some(plans) = &mut summary.plans
|
||||
&& let (Some(plan), Ok(provider)) = (
|
||||
&account.plan_type,
|
||||
providers.require(&account.provider_kind),
|
||||
&& let Some(plan) = providers.resolve_account_plan(
|
||||
account.provider_kind.as_str(),
|
||||
&mut account.plan_type.clone(),
|
||||
quotas.get(&account.id),
|
||||
)
|
||||
{
|
||||
*plans.entry(provider.plan_type_display(plan)).or_default() += 1;
|
||||
*plans.entry(plan).or_default() += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
|
||||
@@ -5,7 +5,7 @@ use super::{
|
||||
};
|
||||
use crate::model::provider_credentials::ProviderQuota;
|
||||
use gateway_core::{
|
||||
account::{QuotaObservation, UpstreamConnection},
|
||||
account::{QuotaObservation, UpstreamConnection, UpstreamInfoFailure},
|
||||
error::StoreError,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
@@ -56,6 +56,7 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling: None,
|
||||
summary: None,
|
||||
unsupported: false,
|
||||
},
|
||||
));
|
||||
}
|
||||
@@ -85,23 +86,25 @@ impl PeerInfoService {
|
||||
connections
|
||||
.iter()
|
||||
.map(|connection| {
|
||||
let cached = observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection);
|
||||
(
|
||||
connection.clone(),
|
||||
observations
|
||||
.get(&connection.account_id)
|
||||
.filter(|cached| cached.connection == *connection)
|
||||
.and_then(|cached| cached.summary.clone()),
|
||||
cached.and_then(|cached| cached.summary.clone()),
|
||||
cached.is_some_and(|cached| cached.unsupported),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
Ok(summaries
|
||||
.into_iter()
|
||||
.map(|(connection, peer)| {
|
||||
.map(|(connection, peer, unsupported)| {
|
||||
let quota = Self::project_peer(
|
||||
&connection,
|
||||
peer.as_deref(),
|
||||
self.last_error(&connection),
|
||||
self.last_error(&connection)
|
||||
.or(unsupported.then_some(UpstreamInfoFailure::Unsupported)),
|
||||
now,
|
||||
);
|
||||
(connection, quota)
|
||||
@@ -132,6 +135,8 @@ impl PeerInfoService {
|
||||
checked_at: now,
|
||||
scheduling,
|
||||
summary,
|
||||
unsupported: observation
|
||||
.is_some_and(|value| snapshot::is_unsupported(&value.quota, connection)),
|
||||
};
|
||||
let previous = self
|
||||
.observations
|
||||
@@ -140,4 +145,12 @@ impl PeerInfoService {
|
||||
.insert(connection.account_id.clone(), cached);
|
||||
drop(previous);
|
||||
}
|
||||
|
||||
pub(super) fn unsupported(&self, connection: &UpstreamConnection) -> bool {
|
||||
self.observations
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&connection.account_id)
|
||||
.is_some_and(|cached| cached.connection == *connection && cached.unsupported)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,7 @@ use crate::model::{
|
||||
},
|
||||
};
|
||||
use gateway_core::account::{QuotaAccessState, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
PeerComponentStatus, PeerLink, PeerPoolDisclosure, PeerQuotaDisclosure,
|
||||
};
|
||||
use gateway_protocol::peer::{PeerComponentStatus, PeerLink};
|
||||
|
||||
use super::{PeerInfoService, quota, snapshot};
|
||||
|
||||
@@ -39,11 +37,11 @@ impl PeerInfoService {
|
||||
last_error: last_error.map(|error| error.as_str().to_owned()),
|
||||
pool: peer
|
||||
.as_ref()
|
||||
.filter(|peer| peer.pool.disclosure == PeerPoolDisclosure::Summary)
|
||||
.and_then(|peer| {
|
||||
let states = peer.pool.states.as_ref()?;
|
||||
Some(UpstreamPool {
|
||||
account_count: peer.pool.account_count?,
|
||||
.and_then(|peer| peer.pool.as_ref())
|
||||
.map(|pool| {
|
||||
let states = &pool.states;
|
||||
UpstreamPool {
|
||||
account_count: pool.account_count,
|
||||
states: UpstreamPoolStates {
|
||||
ready: states.ready,
|
||||
cooldown: states.cooldown,
|
||||
@@ -52,8 +50,7 @@ impl PeerInfoService {
|
||||
error: states.error,
|
||||
unknown: states.unknown,
|
||||
},
|
||||
plans: peer
|
||||
.pool
|
||||
plans: pool
|
||||
.plans
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
@@ -63,19 +60,18 @@ impl PeerInfoService {
|
||||
count: plan.count,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}),
|
||||
};
|
||||
let mut windows = Vec::new();
|
||||
if let Some(peer) = &peer {
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(unit) = &usage.unit
|
||||
{
|
||||
for (key, label, seconds, window) in [
|
||||
("daily", "日额度", 86400, &usage.daily),
|
||||
("weekly", "周额度", 604800, &usage.weekly),
|
||||
] {
|
||||
// 不限额仍有日、周账本,只是不计算占用百分比
|
||||
for window in &usage.windows {
|
||||
// 不限额仍展示已提供的账本窗口,只是不计算占用百分比
|
||||
let total = window
|
||||
.total
|
||||
.as_deref()
|
||||
@@ -84,19 +80,17 @@ impl PeerInfoService {
|
||||
let used_percent = snapshot::decimal(&window.used)
|
||||
.zip(total)
|
||||
.map(|(used, total)| (used / total * 100.0).clamp(0.0, 100.0));
|
||||
let window_observed = peer
|
||||
.usage
|
||||
let window_observed = component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
|| !peer
|
||||
.usage
|
||||
|| !component
|
||||
.observed_at
|
||||
.as_deref()
|
||||
.is_some_and(|value| snapshot::fresh(value, now))
|
||||
|| window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(snapshot::timestamp)
|
||||
.is_some_and(|reset| reset <= now);
|
||||
@@ -106,27 +100,28 @@ impl PeerInfoService {
|
||||
observed_at: window_observed,
|
||||
stale: window_stale,
|
||||
amount: Some(UpstreamQuotaAmount {
|
||||
unit: usage.unit.clone(),
|
||||
unit: unit.clone(),
|
||||
used: window.used.clone(),
|
||||
total: window.total.clone(),
|
||||
remaining: window.remaining.clone(),
|
||||
}),
|
||||
}),
|
||||
key: format!("peer:budget:{key}"),
|
||||
key: format!("peer:budget:{}", window.key),
|
||||
group: "general".to_owned(),
|
||||
label: label.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: Some(if key == "weekly" {
|
||||
ProviderQuotaWindowRole::Secondary
|
||||
} else {
|
||||
ProviderQuotaWindowRole::Primary
|
||||
}),
|
||||
role: match window.key.as_str() {
|
||||
"daily" => Some(ProviderQuotaWindowRole::Primary),
|
||||
"weekly" => Some(ProviderQuotaWindowRole::Secondary),
|
||||
"monthly" => Some(ProviderQuotaWindowRole::Monthly),
|
||||
_ => None,
|
||||
},
|
||||
// Key 消费窗口覆盖此连接的全部请求,本机用量按同一时间边界聚合
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(seconds),
|
||||
window_seconds: window.window_seconds,
|
||||
used_percent,
|
||||
reset_at: window.resets_at.as_deref().and_then(snapshot::datetime),
|
||||
reset_at: window.reset_at.as_deref().and_then(snapshot::datetime),
|
||||
limit_reached: !window_stale
|
||||
&& quota::window_access(window, now) == QuotaAccessState::Exhausted,
|
||||
local_usage: None,
|
||||
@@ -135,8 +130,8 @@ impl PeerInfoService {
|
||||
}
|
||||
}
|
||||
// 池内窗口各自保留账号归属,不能将不同套餐相加为统一消费额度
|
||||
if peer.quota.disclosure == PeerQuotaDisclosure::Accounts {
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
if let Some(quota) = &peer.quota {
|
||||
for (index, account) in quota.accounts.iter().enumerate() {
|
||||
let window_observed =
|
||||
account.observed_at.as_deref().and_then(snapshot::datetime);
|
||||
let window_stale = stale
|
||||
@@ -146,6 +141,7 @@ impl PeerInfoService {
|
||||
now,
|
||||
);
|
||||
for window in &account.windows {
|
||||
let group = window.group.as_deref().unwrap_or("general");
|
||||
let window_stale = window_stale
|
||||
|| window
|
||||
.reset_at
|
||||
@@ -156,7 +152,7 @@ impl PeerInfoService {
|
||||
"peer:{}:{}:{}",
|
||||
account.id.len(),
|
||||
account.id,
|
||||
window.limit_id.as_deref().unwrap_or(&window.group)
|
||||
window.limit_id.as_deref().unwrap_or(group)
|
||||
);
|
||||
windows.push(ProviderQuotaWindow {
|
||||
upstream: Some(UpstreamQuotaWindow {
|
||||
@@ -166,8 +162,8 @@ impl PeerInfoService {
|
||||
amount: None,
|
||||
}),
|
||||
key: format!("{scope}:{}", window.key),
|
||||
group: window.group.clone(),
|
||||
label: window.label.clone(),
|
||||
group: group.to_owned(),
|
||||
label: window.label.as_deref().unwrap_or(&window.key).to_owned(),
|
||||
limit_id: Some(scope),
|
||||
limit_name: Some(match window.limit_name.as_deref() {
|
||||
Some(name) => format!("账号 {} · {name}", index + 1),
|
||||
|
||||
@@ -13,24 +13,26 @@ pub(super) fn observe(
|
||||
started_at: SystemTime,
|
||||
now: SystemTime,
|
||||
) -> Option<QuotaState> {
|
||||
let usage = peer.usage.as_ref()?;
|
||||
if !fresh(&peer.observed_at, now)
|
||||
|| !component_fresh(Some(&peer.observed_at), Some(&peer.expires_at), now)
|
||||
|| peer.usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(peer.usage.observed_at.as_deref(), None, now)
|
||||
|| usage.status != PeerComponentStatus::Available
|
||||
|| !component_fresh(usage.observed_at.as_deref(), None, now)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let observed_at = started_at
|
||||
.min(timestamp(&peer.observed_at)?)
|
||||
.min(timestamp(peer.usage.observed_at.as_deref()?)?);
|
||||
let usage = peer.usage.data.as_ref()?;
|
||||
.min(timestamp(usage.observed_at.as_deref()?)?);
|
||||
let usage = usage.data.as_ref()?;
|
||||
match access(usage, now) {
|
||||
QuotaAccessState::Allowed => Some(QuotaState::allowed(observed_at)),
|
||||
QuotaAccessState::Exhausted => {
|
||||
let reset_at = [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
let reset_at = usage
|
||||
.windows
|
||||
.iter()
|
||||
.filter(|window| window_access(window, now) == QuotaAccessState::Exhausted)
|
||||
.filter_map(|window| window.resets_at.as_deref().and_then(timestamp))
|
||||
.filter_map(|window| window.reset_at.as_deref().and_then(timestamp))
|
||||
.min();
|
||||
Some(QuotaState::exhausted(
|
||||
if usage.allowed == Some(false) {
|
||||
@@ -51,8 +53,9 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
match usage.allowed {
|
||||
Some(true) => QuotaAccessState::Allowed,
|
||||
Some(false) => QuotaAccessState::Exhausted,
|
||||
None if [&usage.daily, &usage.weekly]
|
||||
.into_iter()
|
||||
None if usage
|
||||
.windows
|
||||
.iter()
|
||||
.any(|window| window_access(window, now) == QuotaAccessState::Exhausted) =>
|
||||
{
|
||||
QuotaAccessState::Exhausted
|
||||
@@ -64,7 +67,7 @@ pub(super) fn access(usage: &PeerKeyUsage, now: SystemTime) -> QuotaAccessState
|
||||
/// 过期窗口保持未知,不从重置时间或金额推断恢复
|
||||
pub(super) fn window_access(window: &PeerBudgetWindow, now: SystemTime) -> QuotaAccessState {
|
||||
if window
|
||||
.resets_at
|
||||
.reset_at
|
||||
.as_deref()
|
||||
.and_then(timestamp)
|
||||
.is_some_and(|reset| reset <= now)
|
||||
|
||||
@@ -43,6 +43,9 @@ impl PeerInfoService {
|
||||
}
|
||||
|
||||
pub(super) fn refresh_due(&self, connection: &UpstreamConnection) -> bool {
|
||||
if self.unsupported(connection) {
|
||||
return false;
|
||||
}
|
||||
let state = self.refresh_state(connection);
|
||||
state.try_lock().is_ok_and(|state| {
|
||||
state.connection != *connection || Instant::now() >= state.next_attempt
|
||||
@@ -91,17 +94,18 @@ impl PeerInfoService {
|
||||
.ok_or_else(|| ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))?;
|
||||
let slot = self.refresh_state(&connection);
|
||||
let mut state = slot.lock().await;
|
||||
if state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
if !manual && self.unsupported(&connection)
|
||||
|| state.connection == connection
|
||||
&& Instant::now()
|
||||
< if manual {
|
||||
state.manual_after
|
||||
} else {
|
||||
state.next_attempt
|
||||
}
|
||||
{
|
||||
let error = state.last_error;
|
||||
drop(state);
|
||||
if let Some(error) = error {
|
||||
if let Some(error) = error.filter(|error| *error != UpstreamInfoFailure::Unsupported) {
|
||||
return Err(refresh_error(error));
|
||||
}
|
||||
return self
|
||||
@@ -126,19 +130,18 @@ impl PeerInfoService {
|
||||
let failure = result.as_ref().err().copied();
|
||||
let delay = match failure {
|
||||
None => Duration::from_secs(240 + stable_jitter(id.as_str())),
|
||||
Some(UpstreamInfoFailure::Unauthorized | UpstreamInfoFailure::Unsupported) => {
|
||||
Duration::from_secs(1800)
|
||||
}
|
||||
Some(UpstreamInfoFailure::Unauthorized) => Duration::from_secs(1800),
|
||||
Some(_) => Duration::from_secs(30),
|
||||
}
|
||||
.max(retry_after.unwrap_or_default());
|
||||
state.next_attempt = Instant::now() + delay;
|
||||
state.manual_after = Instant::now()
|
||||
+ if failure.is_some() {
|
||||
+ (if failure.is_some_and(|error| error != UpstreamInfoFailure::Unsupported) {
|
||||
delay
|
||||
} else {
|
||||
Duration::from_secs(15)
|
||||
};
|
||||
})
|
||||
.max(retry_after.unwrap_or_default());
|
||||
state.last_error = failure;
|
||||
let document = match result {
|
||||
Ok(document) => Some(document),
|
||||
@@ -146,6 +149,7 @@ impl PeerInfoService {
|
||||
Err(UpstreamInfoFailure::Unauthorized) => {
|
||||
Some(OpaqueProviderData::new(Default::default()))
|
||||
}
|
||||
Err(UpstreamInfoFailure::Unsupported) => Some(snapshot::unsupported(&connection)),
|
||||
Err(_) => None,
|
||||
};
|
||||
if let Some(document) = document {
|
||||
@@ -185,7 +189,8 @@ impl PeerInfoService {
|
||||
);
|
||||
}
|
||||
drop(state);
|
||||
if let Some(error) = failure {
|
||||
// 不支持协议是正常发现结果,管理端保留分类,后台不重复报警
|
||||
if let Some(error) = failure.filter(|error| *error != UpstreamInfoFailure::Unsupported) {
|
||||
return Err(refresh_error(error));
|
||||
}
|
||||
self.read_quota(id)
|
||||
|
||||
@@ -70,22 +70,23 @@ pub(super) struct SchedulingObservation {
|
||||
impl SchedulingObservation {
|
||||
pub(super) fn new(peer: &PeerLink, now: SystemTime) -> Option<Self> {
|
||||
let validity = Validity::component(Some(&peer.observed_at), Some(&peer.expires_at))?;
|
||||
let mut exhausted_budgets = Vec::with_capacity(2);
|
||||
if peer.usage.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &peer.usage.data
|
||||
let mut exhausted_budgets = Vec::new();
|
||||
if let Some(component) = &peer.usage
|
||||
&& component.status == PeerComponentStatus::Available
|
||||
&& let Some(usage) = &component.data
|
||||
&& let Some(usage_validity) =
|
||||
Validity::component(peer.usage.observed_at.as_deref(), None)
|
||||
Validity::component(component.observed_at.as_deref(), None)
|
||||
&& quota::access(usage, now) == QuotaAccessState::Exhausted
|
||||
{
|
||||
if usage.allowed.is_some() {
|
||||
exhausted_budgets.push(usage_validity);
|
||||
} else {
|
||||
// 窗口提供的补充证据只在各自重置边界之前影响调度评分
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
for window in &usage.windows {
|
||||
if quota::window_access(window, now) == QuotaAccessState::Exhausted {
|
||||
let mut validity = usage_validity;
|
||||
if let Some(reset) =
|
||||
window.resets_at.as_deref().and_then(snapshot::timestamp)
|
||||
window.reset_at.as_deref().and_then(snapshot::timestamp)
|
||||
{
|
||||
validity.expires_at = validity.expires_at.min(reset);
|
||||
}
|
||||
@@ -97,7 +98,11 @@ impl SchedulingObservation {
|
||||
|
||||
// 同一账号重复列出模型不增加候选数,多账号覆盖不计算池平均百分比
|
||||
let mut model_accounts = HashMap::<&str, Option<usize>>::new();
|
||||
for (index, account) in peer.quota.accounts.iter().enumerate() {
|
||||
let quota_accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
for (index, account) in quota_accounts.iter().enumerate() {
|
||||
for model in &account.models {
|
||||
model_accounts
|
||||
.entry(model)
|
||||
@@ -128,7 +133,7 @@ impl SchedulingObservation {
|
||||
accounts
|
||||
.entry(index)
|
||||
.or_insert_with(|| {
|
||||
let account = &peer.quota.accounts[index];
|
||||
let account = "a_accounts[index];
|
||||
let validity = Validity::component(
|
||||
account.observed_at.as_deref(),
|
||||
account.expires_at.as_deref(),
|
||||
|
||||
@@ -12,7 +12,7 @@ use futures::{StreamExt as _, stream};
|
||||
use gateway_core::{
|
||||
account::{
|
||||
LoadedCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
UpstreamConnection, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
UpstreamConnection, UpstreamInfoFailure, UpstreamInfoSource, UpstreamSchedulingSignal,
|
||||
},
|
||||
error::{StoreError, StoreErrorKind},
|
||||
routing::ProviderKind,
|
||||
@@ -34,6 +34,7 @@ pub(super) struct CachedObservation {
|
||||
pub checked_at: SystemTime,
|
||||
pub scheduling: Option<Arc<SchedulingObservation>>,
|
||||
pub summary: Option<Arc<PeerLink>>,
|
||||
pub unsupported: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct PeerInfoService {
|
||||
@@ -171,13 +172,17 @@ impl PeerInfoService {
|
||||
let peer = observation
|
||||
.as_ref()
|
||||
.and_then(|value| snapshot::decode(&value.quota));
|
||||
let unsupported = observation
|
||||
.as_ref()
|
||||
.is_some_and(|value| snapshot::is_unsupported(&value.quota, &connection));
|
||||
if self.current_connection(id).await?.as_ref() != Some(&connection) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Self::project_peer(
|
||||
&connection,
|
||||
peer.as_ref(),
|
||||
self.last_error(&connection),
|
||||
self.last_error(&connection)
|
||||
.or(unsupported.then_some(UpstreamInfoFailure::Unsupported)),
|
||||
SystemTime::now(),
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -1,18 +1,46 @@
|
||||
//! 互联响应的有界校验、能力授权与时间有效性解释
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamInfoFailure};
|
||||
use gateway_core::account::{OpaqueProviderData, UpstreamConnection, UpstreamInfoFailure};
|
||||
use gateway_protocol::peer::{
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerLink, PeerPoolDisclosure,
|
||||
SCHEMA_VERSION,
|
||||
MAX_ACCOUNTS, MAX_MODELS, MAX_QUOTA_WINDOWS, OBJECT, PeerKeyUsage, PeerLink, PeerPool,
|
||||
PeerQuota, SCHEMA_VERSION,
|
||||
};
|
||||
use serde::Deserialize as _;
|
||||
|
||||
pub(super) const MAX_AGE: Duration = Duration::from_secs(30 * 60);
|
||||
pub(super) const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
// 不支持协议是连接的识别结果,复用观测存储与版本保护,不伪造上游额度快照
|
||||
pub(super) fn unsupported(connection: &UpstreamConnection) -> OpaqueProviderData {
|
||||
OpaqueProviderData::new(serde_json::Map::from_iter([
|
||||
("object".into(), "cpr.peer.unsupported".into()),
|
||||
("schema_version".into(), 1.into()),
|
||||
(
|
||||
"connection_revision".into(),
|
||||
connection.revision.clone().into(),
|
||||
),
|
||||
]))
|
||||
}
|
||||
|
||||
pub(super) fn is_unsupported(data: &OpaqueProviderData, connection: &UpstreamConnection) -> bool {
|
||||
let fields = data.expose_to_provider();
|
||||
fields.get("object").and_then(serde_json::Value::as_str) == Some("cpr.peer.unsupported")
|
||||
&& fields
|
||||
.get("schema_version")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
== Some(1)
|
||||
&& fields
|
||||
.get("connection_revision")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(connection.revision.as_str())
|
||||
}
|
||||
|
||||
pub(super) fn timestamp(value: &str) -> Option<SystemTime> {
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.ok()
|
||||
@@ -51,14 +79,15 @@ pub(super) fn component_fresh(
|
||||
}
|
||||
|
||||
pub(super) fn decode(data: &OpaqueProviderData) -> Option<PeerLink> {
|
||||
PeerLink::deserialize(
|
||||
let peer = PeerLink::deserialize(
|
||||
serde::de::value::MapDeserializer::<_, serde_json::Error>::new(
|
||||
data.expose_to_provider()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.as_str(), value)),
|
||||
),
|
||||
)
|
||||
.ok()
|
||||
.ok()?;
|
||||
(peer.object == OBJECT && peer.schema_version == SCHEMA_VERSION).then_some(peer)
|
||||
}
|
||||
|
||||
pub(super) fn parse(
|
||||
@@ -67,11 +96,17 @@ pub(super) fn parse(
|
||||
) -> Result<OpaqueProviderData, UpstreamInfoFailure> {
|
||||
let mut peer: PeerLink =
|
||||
serde_json::from_slice(bytes).map_err(|_| UpstreamInfoFailure::InvalidResponse)?;
|
||||
let valid_text = |value: &str, max: usize| {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
};
|
||||
let observed = timestamp(&peer.observed_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let expires = timestamp(&peer.expires_at).ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let accounts = peer
|
||||
.quota
|
||||
.as_ref()
|
||||
.map_or(&[][..], |quota| quota.accounts.as_slice());
|
||||
let budget_windows = peer
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.data.as_ref())
|
||||
.map_or(0, |usage| usage.windows.len());
|
||||
if peer.object != OBJECT
|
||||
|| peer.schema_version != SCHEMA_VERSION
|
||||
|| observed > now + Duration::from_secs(60)
|
||||
@@ -79,125 +114,39 @@ pub(super) fn parse(
|
||||
|| expires <= observed
|
||||
|| expires > observed + MAX_AGE
|
||||
|| peer.capabilities.len() > 32
|
||||
|| peer
|
||||
.quota
|
||||
.accounts
|
||||
|| accounts
|
||||
.iter()
|
||||
.map(|account| account.windows.len())
|
||||
.sum::<usize>()
|
||||
> MAX_QUOTA_WINDOWS
|
||||
|| peer.quota.accounts.len() > MAX_ACCOUNTS
|
||||
|| budget_windows > MAX_QUOTA_WINDOWS
|
||||
|| accounts.len() > MAX_ACCOUNTS
|
||||
|| peer.models.len() > MAX_MODELS
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
// 未授权能力不能借同名数据块越过逐层公开边界
|
||||
if !peer.capabilities.iter().any(|value| value == "key_usage") {
|
||||
peer.usage.data = None;
|
||||
let disclosed = |capability| peer.capabilities.iter().any(|value| value == capability);
|
||||
if !disclosed("key_usage") {
|
||||
peer.usage = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "pool_summary")
|
||||
{
|
||||
peer.pool = Default::default();
|
||||
if !disclosed("pool_summary") {
|
||||
peer.pool = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "quota_windows")
|
||||
{
|
||||
peer.quota = Default::default();
|
||||
if !disclosed("quota_windows") {
|
||||
peer.quota = None;
|
||||
}
|
||||
if !peer
|
||||
.capabilities
|
||||
.iter()
|
||||
.any(|value| value == "model_availability")
|
||||
{
|
||||
if !disclosed("model_availability") {
|
||||
peer.models.clear();
|
||||
}
|
||||
if peer.pool.disclosure == PeerPoolDisclosure::Summary {
|
||||
let count = peer
|
||||
.pool
|
||||
.account_count
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let states = peer
|
||||
.pool
|
||||
.states
|
||||
.as_ref()
|
||||
.ok_or(UpstreamInfoFailure::InvalidResponse)?;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &peer.pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(pool) = &peer.pool {
|
||||
validate_pool(pool)?;
|
||||
}
|
||||
if let Some(usage) = &peer.usage.data {
|
||||
if !valid_text(&usage.unit, 16) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
for window in [&usage.daily, &usage.weekly] {
|
||||
if [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.resets_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
if let Some(usage) = peer.usage.as_ref().and_then(|usage| usage.data.as_ref()) {
|
||||
validate_usage(usage)?;
|
||||
}
|
||||
for account in &peer.quota.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| !valid_text(&window.label, 256)
|
||||
|| !valid_text(&window.group, 128)
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(quota) = &peer.quota {
|
||||
validate_quota(quota)?;
|
||||
}
|
||||
for model in &peer.models {
|
||||
if !valid_text(&model.model, 256)
|
||||
@@ -219,6 +168,113 @@ pub(super) fn parse(
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_text(value: &str, max: usize) -> bool {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
}
|
||||
|
||||
fn validate_pool(pool: &PeerPool) -> Result<(), UpstreamInfoFailure> {
|
||||
let count = pool.account_count;
|
||||
let states = &pool.states;
|
||||
let state_total = [
|
||||
states.ready,
|
||||
states.cooldown,
|
||||
states.exhausted,
|
||||
states.disabled,
|
||||
states.error,
|
||||
states.unknown,
|
||||
]
|
||||
.into_iter()
|
||||
.map(u64::from)
|
||||
.sum::<u64>();
|
||||
if u64::from(count) > MAX_ACCOUNTS as u64 || state_total != u64::from(count) {
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
if let Some(plans) = &pool.plans
|
||||
&& (plans.len() > MAX_ACCOUNTS
|
||||
|| plans
|
||||
.iter()
|
||||
.any(|plan| !valid_text(&plan.name, 256) || plan.count > count)
|
||||
|| plans.iter().map(|plan| u64::from(plan.count)).sum::<u64>() > u64::from(count))
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_usage(usage: &PeerKeyUsage) -> Result<(), UpstreamInfoFailure> {
|
||||
if usage
|
||||
.unit
|
||||
.as_deref()
|
||||
.is_some_and(|unit| !valid_text(unit, 16))
|
||||
|| !usage.windows.is_empty() && usage.unit.is_none()
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
let mut keys = HashSet::new();
|
||||
for window in &usage.windows {
|
||||
if !valid_text(&window.key, 256)
|
||||
|| !keys.insert(&window.key)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window.window_seconds == Some(0)
|
||||
|| [
|
||||
Some(&window.used),
|
||||
window.total.as_ref(),
|
||||
window.remaining.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| decimal(value).is_none())
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_quota(quota: &PeerQuota) -> Result<(), UpstreamInfoFailure> {
|
||||
for account in "a.accounts {
|
||||
if !valid_text(&account.id, 256)
|
||||
|| account.models.len() > MAX_MODELS
|
||||
|| account.windows.iter().any(|window| {
|
||||
!valid_text(&window.key, 256)
|
||||
|| window
|
||||
.label
|
||||
.as_deref()
|
||||
.is_some_and(|label| !valid_text(label, 256))
|
||||
|| window
|
||||
.group
|
||||
.as_deref()
|
||||
.is_some_and(|group| !valid_text(group, 128))
|
||||
|| window
|
||||
.limit_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_text(value, 256))
|
||||
|| window
|
||||
.used_percent
|
||||
.is_some_and(|value| !value.is_finite() || !(0.0..=100.0).contains(&value))
|
||||
|| window
|
||||
.reset_at
|
||||
.as_ref()
|
||||
.is_some_and(|value| timestamp(value).is_none())
|
||||
})
|
||||
{
|
||||
return Err(UpstreamInfoFailure::InvalidResponse);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn decimal(value: &str) -> Option<f64> {
|
||||
if value.is_empty()
|
||||
|| value.len() > 64
|
||||
|
||||
@@ -217,7 +217,7 @@ impl PluginsService {
|
||||
artifact_sha256: metadata.sha256.clone(),
|
||||
enabled: !configuration_required && compatible,
|
||||
configuration: configuration.clone(),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: bindings.clone(),
|
||||
};
|
||||
match self.configure_instance(None, input, context).await {
|
||||
|
||||
@@ -28,17 +28,16 @@ pub(super) fn same_creation_request(
|
||||
existing: &PluginInstance,
|
||||
input: &ConfigurePluginInstance,
|
||||
) -> bool {
|
||||
let same_secrets = input
|
||||
let mut requested_secrets = input
|
||||
.secrets
|
||||
.as_ref()
|
||||
.map_or(existing.secrets.is_empty(), |secrets| {
|
||||
secrets.len() == existing.secrets.len()
|
||||
&& secrets.iter().all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
})
|
||||
.iter()
|
||||
.filter_map(|(key, value)| value.as_ref().map(|value| (key, value)));
|
||||
let same_secrets = requested_secrets.clone().count() == existing.secrets.len()
|
||||
&& requested_secrets.all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
});
|
||||
existing.name == input.name.trim()
|
||||
&& existing.artifact_sha256 == input.artifact_sha256
|
||||
|
||||
@@ -158,7 +158,6 @@ impl PluginsService {
|
||||
}
|
||||
let saved = if let Some(previous) = &existing
|
||||
&& previous.artifact_sha256 != input.artifact_sha256
|
||||
&& input.secrets.is_none()
|
||||
{
|
||||
self.store
|
||||
.load_version_configuration(&previous.id, &input.artifact_sha256)
|
||||
@@ -167,6 +166,21 @@ impl PluginsService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut secrets = saved.map(|saved| saved.secrets).unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
});
|
||||
for (name, value) in input.secrets {
|
||||
match value {
|
||||
Some(value) => {
|
||||
secrets.insert(name, value);
|
||||
}
|
||||
None => {
|
||||
secrets.remove(&name);
|
||||
}
|
||||
}
|
||||
}
|
||||
let instance = PluginInstance {
|
||||
id: existing.as_ref().map_or_else(
|
||||
|| {
|
||||
@@ -182,14 +196,7 @@ impl PluginsService {
|
||||
enabled: input.enabled,
|
||||
trusted_process: true,
|
||||
configuration: input.configuration,
|
||||
secrets: input
|
||||
.secrets
|
||||
.or_else(|| saved.map(|saved| saved.secrets))
|
||||
.unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
}),
|
||||
secrets,
|
||||
bindings: input.bindings,
|
||||
revision: candidate_revision,
|
||||
};
|
||||
|
||||
@@ -366,6 +366,7 @@ fn target_environment_matches(target: &str) -> bool {
|
||||
match target {
|
||||
"x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => cfg!(target_env = "gnu"),
|
||||
"aarch64-apple-darwin" => cfg!(target_os = "macos"),
|
||||
"x86_64-pc-windows-msvc" | "aarch64-pc-windows-msvc" => cfg!(target_env = "msvc"),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -392,4 +393,18 @@ const SUPPORTED_PLATFORMS: &[Platform] = &[
|
||||
runtime_os: "macos",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "amd64",
|
||||
target: "x86_64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "x86_64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "arm64",
|
||||
target: "aarch64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -69,7 +69,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: saved.configuration,
|
||||
secrets: Some(saved.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: saved.bindings,
|
||||
};
|
||||
// 沿用同一份快照的 CAS 和状态迁移流程,不能在检查目标后重新读取并覆盖并发配置
|
||||
@@ -202,7 +202,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: configuration.configuration,
|
||||
secrets: Some(configuration.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: configuration.bindings,
|
||||
};
|
||||
// 同一次读取的 CAS 覆盖草稿计算、准备与提交,不能覆盖并发设置
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::plugin_update::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use super::system_preflight::{ConfirmedPluginRestart, PluginSystemUpdatePreflight};
|
||||
use crate::model::{MutationContext, system::SystemRestartPlan};
|
||||
use async_trait::async_trait;
|
||||
|
||||
|
||||
@@ -1166,6 +1166,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -41,6 +41,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -11,11 +11,11 @@ mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod peer_link;
|
||||
mod plugin_update;
|
||||
mod plugins;
|
||||
mod proxies;
|
||||
mod settings;
|
||||
mod system;
|
||||
mod system_preflight;
|
||||
mod usage_data;
|
||||
|
||||
use std::{
|
||||
|
||||
@@ -1384,6 +1384,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -1478,6 +1479,7 @@ fn total_record(
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 公共互联按 Provider 声明发现连接,不将认证种类或调度观测绑在 OpenAI 上
|
||||
|
||||
mod refresh;
|
||||
|
||||
use super::super::UnavailableProvider;
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
@@ -390,14 +392,13 @@ impl PeerTransport for TestTransport {
|
||||
.push(account.id().as_str().to_owned());
|
||||
let now = Utc::now();
|
||||
Ok(serde_json::to_vec(&serde_json::json!({
|
||||
"object":"cpr.peer", "schema_version":1,
|
||||
"object":"cpr.peer", "schema_version":2,
|
||||
"observed_at":now.to_rfc3339(),
|
||||
"expires_at":(now + chrono::Duration::minutes(5)).to_rfc3339(),
|
||||
"capabilities":["key_usage"],
|
||||
"usage":{"status":"available","observed_at":now.to_rfc3339(),"data":{"unit":"USD","allowed":false,
|
||||
"daily":{"total":"1","used":"1","remaining":"0","resetsAt":(now + chrono::Duration::days(1)).to_rfc3339()},
|
||||
"weekly":{"total":"0","used":"0","remaining":"0"}}},
|
||||
"pool":{"disclosure":"none"}, "quota":{"disclosure":"none"}, "models":[]
|
||||
"windows":[{"key":"daily","total":"1","used":"1","remaining":"0","reset_at":(now + chrono::Duration::days(1)).to_rfc3339()}]}},
|
||||
"models":[]
|
||||
})).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
//! 互联发现结果的持久恢复、连接失效、手动重查与临时失败退避
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::PeerUpstreamBundle;
|
||||
use gateway_core::task::WorkerRegistration;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_survives_restart_until_manual_rediscovery() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
// 入库后的首查与手动刷新使用同一入口,不支持协议也是可保存的正常发现结果
|
||||
let initial = bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(!initial.upstream.unwrap().discovered);
|
||||
let first_worker = worker(&mut bundle);
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&first_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
drop(first_worker);
|
||||
drop(bundle);
|
||||
|
||||
let mut restarted = initialize(accounts, transport.clone());
|
||||
let info = restarted
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert!(!info.discovered);
|
||||
assert_eq!(info.last_error.as_deref(), Some("unsupported"));
|
||||
let resumed_worker = worker(&mut restarted);
|
||||
cycle(&resumed_worker).await;
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
let summary = restarted
|
||||
.service()
|
||||
.read_summary(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
summary.upstream.unwrap().last_error.as_deref(),
|
||||
Some("unsupported")
|
||||
);
|
||||
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
let discovered = restarted.service().refresh_quota(&id).await.unwrap();
|
||||
assert!(discovered.upstream.unwrap().discovered);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
tokio::time::advance(Duration::from_secs(360)).await;
|
||||
cycle(&resumed_worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_detection_keeps_manual_refresh_throttling() {
|
||||
for retry_after in [None, Some(Duration::from_secs(90))] {
|
||||
let (bundle, _, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
*transport.failure.lock().unwrap() = Some(PeerFetchFailure {
|
||||
kind: UpstreamInfoFailure::Unsupported,
|
||||
retry_after,
|
||||
});
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
let delay = retry_after.unwrap_or(Duration::from_secs(15));
|
||||
tokio::time::advance(delay - Duration::from_secs(1)).await;
|
||||
bundle.service().refresh_quota(&id).await.unwrap();
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.refresh_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn ordinary_edits_keep_detection_but_proxy_and_credential_changes_reprobe() {
|
||||
let (mut bundle, accounts, transport, id) = fixture(UpstreamInfoFailure::Unsupported);
|
||||
let worker = worker(&mut bundle);
|
||||
// 无识别记录的存量账号由周期任务补查
|
||||
cycle(&worker).await;
|
||||
let initial = accounts.accounts.lock().unwrap()[0].clone();
|
||||
let edited = initial.clone().with_notes(Some("修改备注".into()));
|
||||
*accounts.accounts.lock().unwrap() = vec![edited.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(86400)).await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
|
||||
let reconnected = edited.with_outbound_proxy(Some(
|
||||
OutboundProxy::parse("http://proxy.invalid:8080").unwrap(),
|
||||
));
|
||||
*accounts.accounts.lock().unwrap() = vec![reconnected.clone()];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
|
||||
let replaced = ProviderAccount::new(
|
||||
id.clone(),
|
||||
initial.provider().clone(),
|
||||
"new credential".into(),
|
||||
None,
|
||||
"peer_test".into(),
|
||||
CredentialRevision::new(2).unwrap(),
|
||||
None,
|
||||
)
|
||||
.with_outbound_proxy(reconnected.outbound_proxy().cloned())
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
*accounts.accounts.lock().unwrap() = vec![replaced];
|
||||
bundle.service().account_changed(&id).await.unwrap();
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn uncertain_discovery_failures_retry_without_becoming_unsupported() {
|
||||
for failure in [
|
||||
UpstreamInfoFailure::Unauthorized,
|
||||
UpstreamInfoFailure::Unavailable,
|
||||
UpstreamInfoFailure::RateLimited,
|
||||
UpstreamInfoFailure::InvalidResponse,
|
||||
] {
|
||||
let (mut bundle, _, transport, id) = fixture(failure);
|
||||
let worker = worker(&mut bundle);
|
||||
cycle(&worker).await;
|
||||
let info = bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap();
|
||||
assert_eq!(info.last_error.as_deref(), Some(failure.as_str()));
|
||||
assert!(!info.discovered);
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 1);
|
||||
*transport.failure.lock().unwrap() = None;
|
||||
tokio::time::advance(Duration::from_secs(
|
||||
if failure == UpstreamInfoFailure::Unauthorized {
|
||||
1800
|
||||
} else {
|
||||
30
|
||||
},
|
||||
))
|
||||
.await;
|
||||
cycle(&worker).await;
|
||||
assert_eq!(transport.requests.load(Ordering::SeqCst), 2);
|
||||
assert!(
|
||||
bundle
|
||||
.service()
|
||||
.read_quota(&id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.upstream
|
||||
.unwrap()
|
||||
.discovered
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(
|
||||
failure: UpstreamInfoFailure,
|
||||
) -> (
|
||||
PeerUpstreamBundle,
|
||||
Arc<TestAccounts>,
|
||||
Arc<DetectionTransport>,
|
||||
ProviderAccountId,
|
||||
) {
|
||||
let record = account("acct_peer", "openai", "peer_test");
|
||||
let id = record.id().clone();
|
||||
let accounts = Arc::new(TestAccounts {
|
||||
accounts: Mutex::new(vec![record]),
|
||||
records: Mutex::default(),
|
||||
failed_provider: None,
|
||||
});
|
||||
let transport = Arc::new(DetectionTransport {
|
||||
requests: AtomicUsize::new(0),
|
||||
failure: Mutex::new(Some(failure.into())),
|
||||
success: TestTransport(Mutex::default()),
|
||||
});
|
||||
(
|
||||
initialize(accounts.clone(), transport.clone()),
|
||||
accounts,
|
||||
transport,
|
||||
id,
|
||||
)
|
||||
}
|
||||
|
||||
fn initialize(
|
||||
accounts: Arc<TestAccounts>,
|
||||
transport: Arc<DetectionTransport>,
|
||||
) -> PeerUpstreamBundle {
|
||||
let mut provider = UnavailableProvider::new("openai");
|
||||
provider.peer_enabled = true;
|
||||
let registry =
|
||||
ProviderAdminRegistry::new([Arc::new(provider) as Arc<dyn ProviderAdmin>]).unwrap();
|
||||
initialize_peer_upstream(accounts, ®istry, transport).unwrap()
|
||||
}
|
||||
|
||||
fn worker(bundle: &mut PeerUpstreamBundle) -> WorkerRegistration {
|
||||
let WorkerContribution::Registration(worker) = bundle.take_worker_contributions().remove(0)
|
||||
else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
worker
|
||||
}
|
||||
|
||||
async fn cycle(worker: &WorkerRegistration) {
|
||||
let WorkerRunnable::Scheduled { task, .. } = &worker.runnable else {
|
||||
panic!("scheduled worker")
|
||||
};
|
||||
task.run_cycle(WorkerCycleContext::new(
|
||||
worker.id.clone(),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
struct DetectionTransport {
|
||||
requests: AtomicUsize,
|
||||
failure: Mutex<Option<PeerFetchFailure>>,
|
||||
success: TestTransport,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl PeerTransport for DetectionTransport {
|
||||
async fn fetch(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
connection: &PeerConnection,
|
||||
) -> Result<Vec<u8>, PeerFetchFailure> {
|
||||
self.requests.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(failure) = self.failure.lock().unwrap().as_ref() {
|
||||
return Err(PeerFetchFailure {
|
||||
kind: failure.kind,
|
||||
retry_after: failure.retry_after,
|
||||
});
|
||||
}
|
||||
self.success.fetch(account, connection).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
//! 实例敏感配置的逐字段更新、版本基准与提交隔离测试
|
||||
|
||||
use super::*;
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
fn secret_values(instance: &PluginInstance) -> serde_json::Value {
|
||||
serde_json::Value::Object(
|
||||
instance
|
||||
.secrets
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), json!(value.expose_secret())))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn secret_changes_apply_to_current_or_target_version_configuration() {
|
||||
for (target, saved_version) in [
|
||||
(OLD_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, true),
|
||||
] {
|
||||
let token = if saved_version {
|
||||
"saved-token"
|
||||
} else {
|
||||
"current-token"
|
||||
};
|
||||
let untouched = if saved_version {
|
||||
"saved-value"
|
||||
} else {
|
||||
"current-value"
|
||||
};
|
||||
for (changes, expected) in [
|
||||
(json!({}), json!({"token":token,"untouched":untouched})),
|
||||
(
|
||||
json!({"adminKey":"new-key"}),
|
||||
json!({"token":token,"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
(
|
||||
json!({"token":"new-token"}),
|
||||
json!({"token":"new-token","untouched":untouched}),
|
||||
),
|
||||
(json!({"token":null}), json!({"untouched":untouched})),
|
||||
(
|
||||
json!({"token":null,"adminKey":"new-key","absent":null}),
|
||||
json!({"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
{
|
||||
let mut data = fixture.data.lock().unwrap();
|
||||
data.snapshot.instances[0].secrets = BTreeMap::from([
|
||||
("token".into(), "current-token".into()),
|
||||
("untouched".into(), "current-value".into()),
|
||||
]);
|
||||
if saved_version {
|
||||
data.history.get_mut(NEW_ARTIFACT).unwrap().secrets = BTreeMap::from([
|
||||
("token".into(), "saved-token".into()),
|
||||
("untouched".into(), "saved-value".into()),
|
||||
]);
|
||||
} else {
|
||||
data.history.clear();
|
||||
}
|
||||
}
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = target.into();
|
||||
request.expected_revision = Some(10);
|
||||
request.secrets = serde_json::from_value(changes).unwrap();
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(secret_values(&result.instance), expected);
|
||||
assert_eq!(secret_values(&fixture.snapshot().instances[0]), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_secret_changes_leave_saved_values_unchanged() {
|
||||
for failure in ["stale", "preparation", "size"] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].secrets =
|
||||
BTreeMap::from([("token".into(), "saved-token".into())]);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
request.expected_revision = Some(if failure == "stale" { 9 } else { 10 });
|
||||
request.secrets = BTreeMap::from([
|
||||
("token".into(), None),
|
||||
(
|
||||
"adminKey".into(),
|
||||
Some(
|
||||
if failure == "size" {
|
||||
"x".repeat(8193)
|
||||
} else {
|
||||
"new-key".into()
|
||||
}
|
||||
.into(),
|
||||
),
|
||||
),
|
||||
]);
|
||||
fixture.data.lock().unwrap().fail_preparation = failure == "preparation";
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await;
|
||||
assert!(result.is_err(), "{failure} must reject the update");
|
||||
assert_eq!(
|
||||
secret_values(&fixture.snapshot().instances[0]),
|
||||
secret_values(&before.instances[0])
|
||||
);
|
||||
assert!(fixture.data.lock().unwrap().saves.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ use serde_json::json;
|
||||
|
||||
use super::TestPluginPorts;
|
||||
|
||||
mod configuration;
|
||||
mod queries;
|
||||
|
||||
const OLD_ARTIFACT: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
@@ -759,7 +760,7 @@ fn input() -> ConfigurePluginInstance {
|
||||
artifact_sha256: NEW_ARTIFACT.into(),
|
||||
enabled: true,
|
||||
configuration: json!({}),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: vec![],
|
||||
}
|
||||
}
|
||||
@@ -942,6 +943,8 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
// 此用例验证创建重试,使用不需要私有状态迁移的固定版本
|
||||
value.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
value.enabled = false;
|
||||
value.secrets =
|
||||
serde_json::from_value(json!({"token": "saved-token", "absent": null})).unwrap();
|
||||
value
|
||||
};
|
||||
let created = service
|
||||
@@ -955,6 +958,12 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
assert_eq!(created.instance.id, creation_id);
|
||||
assert_eq!(retried.instance.id, creation_id);
|
||||
assert_eq!(fixture.snapshot().instances.len(), 2);
|
||||
use secrecy::ExposeSecret as _;
|
||||
assert_eq!(
|
||||
retried.instance.secrets["token"].expose_secret(),
|
||||
"saved-token"
|
||||
);
|
||||
assert!(!retried.instance.secrets.contains_key("absent"));
|
||||
let before = fixture.snapshot().config_revision;
|
||||
let mut changed = request();
|
||||
changed.name = "changed draft".into();
|
||||
@@ -965,6 +974,17 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
let mut changed = request();
|
||||
changed
|
||||
.secrets
|
||||
.insert("token".into(), Some("different-token".into()));
|
||||
let error = service
|
||||
.configure_instance(None, changed, &context())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1435,23 +1455,6 @@ async fn version_switch_preparation_failure_preserves_current_and_saved_settings
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_edit_preserves_the_target_versions_secrets_when_omitted() {
|
||||
use secrecy::ExposeSecret as _;
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.secrets = None;
|
||||
let result = service(fixture, Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.instance.secrets["token"].expose_secret(),
|
||||
"old-version-secret"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_requires_a_saved_configuration_instead_of_reusing_incompatible_current_settings()
|
||||
{
|
||||
|
||||
@@ -446,6 +446,8 @@ fn release_platform() -> (&'static str, &'static str, &'static str) {
|
||||
("linux", "x86_64") => ("linux", "amd64", "x86_64-unknown-linux-gnu"),
|
||||
("linux", "aarch64") => ("linux", "arm64", "aarch64-unknown-linux-gnu"),
|
||||
("macos", "aarch64") => ("darwin", "arm64", "aarch64-apple-darwin"),
|
||||
("windows", "x86_64") => ("windows", "amd64", "x86_64-pc-windows-msvc"),
|
||||
("windows", "aarch64") => ("windows", "arm64", "aarch64-pc-windows-msvc"),
|
||||
platform => panic!("unsupported test platform: {platform:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +334,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -271,6 +271,7 @@ pub(crate) fn usage_list_record_view(
|
||||
account_notes: record.provider_account_notes,
|
||||
account_plan_type: record.provider_account_plan_type,
|
||||
account_plan_type_display: record.provider_account_plan_type_display,
|
||||
account_upstream_peer: record.provider_account_upstream_peer,
|
||||
route: record.endpoint,
|
||||
model,
|
||||
requested_model: record.requested_model_id,
|
||||
@@ -1351,6 +1352,7 @@ pub(crate) fn diagnostics_view(
|
||||
request_count: item.request_count,
|
||||
account_plan_type: item.account_plan_type,
|
||||
account_plan_type_display: item.account_plan_type_display,
|
||||
account_upstream_peer: item.account_upstream_peer,
|
||||
success_count: item.success_count,
|
||||
error_count: item.error_count,
|
||||
error_rate: item.error_rate,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
pub use crate::time::parse_datetime;
|
||||
|
||||
/// 观测列表默认页大小
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小
|
||||
@@ -256,16 +258,6 @@ impl DiagnosticDimension {
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = trimmed(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
/// 解析 HTTP 状态码
|
||||
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
|
||||
value
|
||||
|
||||
@@ -99,6 +99,7 @@ pub struct UsageListRecordView {
|
||||
pub account_notes: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
@@ -849,6 +850,7 @@ pub struct DiagnosticItemView {
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub error_count: u64,
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -751,6 +752,7 @@ impl RuntimeSettingsFields {
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: self.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: self.openai_account_affinity,
|
||||
max_account_rotations: self.max_account_rotations,
|
||||
@@ -790,6 +792,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: settings.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
|
||||
@@ -9,25 +9,7 @@ use axum::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// 管理端 wire 字段校验错误;只携带稳定字段名,不回显输入值
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WireValidationError {
|
||||
field: &'static str,
|
||||
}
|
||||
|
||||
impl WireValidationError {
|
||||
/// 构造字段校验错误
|
||||
#[must_use]
|
||||
pub const fn new(field: &'static str) -> Self {
|
||||
Self { field }
|
||||
}
|
||||
|
||||
/// 返回未通过校验的字段名
|
||||
#[must_use]
|
||||
pub const fn field(self) -> &'static str {
|
||||
self.field
|
||||
}
|
||||
}
|
||||
pub use crate::WireValidationError;
|
||||
|
||||
/// 成功响应的稳定业务码
|
||||
pub const ADMIN_OK_CODE: u32 = 200;
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! 数据面与公共 HTTP 中间件共用的客户端错误合同和重试投影
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::{HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// OpenAI 风格的协议错误 JSON
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolErrorBody {
|
||||
/// 错误对象
|
||||
pub error: ProtocolError,
|
||||
}
|
||||
|
||||
/// 不包含请求正文或 prompt 的错误对象
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolError {
|
||||
/// OpenAI 错误类别
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
/// 稳定机器码
|
||||
pub code: &'static str,
|
||||
/// 安全的可读信息
|
||||
pub message: String,
|
||||
/// 出错字段;不包含字段值
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
}
|
||||
|
||||
impl ProtocolErrorBody {
|
||||
/// 序列化为 JSON value
|
||||
#[must_use]
|
||||
pub fn into_value(self) -> Value {
|
||||
serde_json::json!({ "error": self.error })
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
pub fn openai_error_response(
|
||||
status: StatusCode,
|
||||
message: &str,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
) -> (StatusCode, Json<Value>) {
|
||||
let body = json!({
|
||||
"error": {"message": message, "type": error_type, "code": code}
|
||||
});
|
||||
match capacity_error_for_client(&body) {
|
||||
Some(body) => (StatusCode::SERVICE_UNAVAILABLE, Json(body)),
|
||||
None => (status, Json(body)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(crate) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
let capacity_code = |value: &Value| {
|
||||
value
|
||||
.as_str()
|
||||
.is_some_and(|code| client_error_code(code) != code)
|
||||
};
|
||||
if !CODE_PATHS
|
||||
.iter()
|
||||
.any(|path| data.pointer(path).is_some_and(capacity_code))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
{
|
||||
*code = Value::String("server_error".to_owned());
|
||||
}
|
||||
}
|
||||
// WS 包装错误按 HTTP 状态分类;保留 400/429 会使客户端终止重试
|
||||
for field in ["status", "status_code"] {
|
||||
if let Some(status) = projected.get_mut(field)
|
||||
&& status.is_number()
|
||||
{
|
||||
*status = json!(503);
|
||||
}
|
||||
}
|
||||
Some(projected)
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
let mut response = openai_error_response(
|
||||
status,
|
||||
error.client_message(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(delay) = error.retry_after()
|
||||
&& let Ok(value) = HeaderValue::from_str(
|
||||
&delay
|
||||
.as_secs()
|
||||
.saturating_add(u64::from(delay.subsec_nanos() > 0))
|
||||
.max(1)
|
||||
.to_string(),
|
||||
)
|
||||
{
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
kind: GatewayErrorKind,
|
||||
) -> (StatusCode, &'static str, &'static str) {
|
||||
match kind {
|
||||
GatewayErrorKind::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
),
|
||||
GatewayErrorKind::Unsupported => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"unsupported_capability",
|
||||
),
|
||||
GatewayErrorKind::Unauthorized => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
),
|
||||
GatewayErrorKind::PolicyDenied => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"invalid_request_error",
|
||||
"policy_denied",
|
||||
),
|
||||
GatewayErrorKind::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
),
|
||||
GatewayErrorKind::NoAvailableProvider => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_full",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_timeout",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
),
|
||||
GatewayErrorKind::BudgetExhausted => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"usage_limit_reached",
|
||||
"usage_limit_reached",
|
||||
),
|
||||
GatewayErrorKind::UpstreamUnavailable => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
"request_timeout",
|
||||
),
|
||||
GatewayErrorKind::Cancelled => (
|
||||
StatusCode::REQUEST_TIMEOUT,
|
||||
"server_error",
|
||||
"request_cancelled",
|
||||
),
|
||||
GatewayErrorKind::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ use crate::openai::service::OpenAiService;
|
||||
|
||||
pub mod admin;
|
||||
pub mod auth;
|
||||
mod client_error;
|
||||
mod health;
|
||||
mod key_usage;
|
||||
mod middleware;
|
||||
@@ -40,6 +41,26 @@ mod time;
|
||||
|
||||
pub use time::{RequestBucketView, TimePresenter};
|
||||
|
||||
/// HTTP wire 字段校验错误;只携带稳定字段名,不回显输入值
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WireValidationError {
|
||||
field: &'static str,
|
||||
}
|
||||
|
||||
impl WireValidationError {
|
||||
/// 构造字段校验错误
|
||||
#[must_use]
|
||||
pub const fn new(field: &'static str) -> Self {
|
||||
Self { field }
|
||||
}
|
||||
|
||||
/// 返回未通过校验的字段名
|
||||
#[must_use]
|
||||
pub const fn field(self) -> &'static str {
|
||||
self.field
|
||||
}
|
||||
}
|
||||
|
||||
/// API-owned HTTP 与静态资源配置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct ApiConfig {
|
||||
|
||||
@@ -262,7 +262,7 @@ async fn handle(
|
||||
Err(MiddlewareError::Gateway(error))
|
||||
if error.kind() == GatewayErrorKind::MiddlewareCapacityUnavailable =>
|
||||
{
|
||||
crate::openai::error::gateway_error_response(&error)
|
||||
crate::client_error::gateway_error_response(&error)
|
||||
}
|
||||
Err(error) => {
|
||||
let status = if error.is_rejected() {
|
||||
|
||||
@@ -15,7 +15,8 @@ use gateway_core::{
|
||||
use gateway_protocol::openai::sse::{DONE_SSE_FRAME, SseEvent, SseEventDecoder, sse_frame_is_done};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::openai::{middleware::error_response, responses::ProtocolErrorBody};
|
||||
use crate::client_error::ProtocolErrorBody;
|
||||
use crate::openai::middleware::error_response;
|
||||
|
||||
use super::response::{ChatIdentity, ChatStreamEncoder, project_response};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use gateway_protocol::openai::sse::encode_sse_event;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::openai::responses::{ProtocolError, ProtocolErrorBody};
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(in crate::openai) struct ChatIdentity {
|
||||
|
||||
@@ -16,6 +16,7 @@ use gateway_core::{
|
||||
operation::Operation,
|
||||
};
|
||||
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
use crate::middleware::headers::encode_headers;
|
||||
use crate::openai::middleware::{
|
||||
ExpectedBody, PendingExecution, RequestInput, buffered_response, error_response,
|
||||
@@ -25,7 +26,6 @@ use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
use super::{
|
||||
error::{engine_error_response, gateway_error_response, protocol_error_response},
|
||||
responses::{ProtocolError, ProtocolErrorBody},
|
||||
service::OpenAiService,
|
||||
};
|
||||
|
||||
|
||||
@@ -8,72 +8,18 @@ use axum::{
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::EngineError,
|
||||
error::{GatewayError, GatewayErrorKind, ProviderErrorKind},
|
||||
error::ProviderErrorKind,
|
||||
policy::{ClientVersionRejection, CodexClientKind},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
|
||||
use super::responses::ProtocolErrorBody;
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client};
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
pub fn openai_error_response(
|
||||
status: StatusCode,
|
||||
message: &str,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
) -> (StatusCode, Json<Value>) {
|
||||
let body = json!({
|
||||
"error": {"message": message, "type": error_type, "code": code}
|
||||
});
|
||||
match capacity_error_for_client(&body) {
|
||||
Some(body) => (StatusCode::SERVICE_UNAVAILABLE, Json(body)),
|
||||
None => (status, Json(body)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(super) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(super) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
let capacity_code = |value: &Value| {
|
||||
value
|
||||
.as_str()
|
||||
.is_some_and(|code| client_error_code(code) != code)
|
||||
};
|
||||
if !CODE_PATHS
|
||||
.iter()
|
||||
.any(|path| data.pointer(path).is_some_and(capacity_code))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
{
|
||||
*code = Value::String("server_error".to_owned());
|
||||
}
|
||||
}
|
||||
// WS 包装错误按 HTTP 状态分类;保留 400/429 会使客户端终止重试
|
||||
for field in ["status", "status_code"] {
|
||||
if let Some(status) = projected.get_mut(field)
|
||||
&& status.is_number()
|
||||
{
|
||||
*status = json!(503);
|
||||
}
|
||||
}
|
||||
Some(projected)
|
||||
}
|
||||
pub use crate::client_error::{
|
||||
gateway_error_contract, gateway_error_response, openai_error_response,
|
||||
};
|
||||
|
||||
/// 严格协议 decoder/encoder 返回的安全错误 body
|
||||
pub fn protocol_error_response(status: StatusCode, body: ProtocolErrorBody) -> Response {
|
||||
@@ -151,32 +97,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
)
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
let mut response = openai_error_response(
|
||||
status,
|
||||
error.client_message(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(delay) = error.retry_after()
|
||||
&& let Ok(value) = HeaderValue::from_str(
|
||||
&delay
|
||||
.as_secs()
|
||||
.saturating_add(u64::from(delay.subsec_nanos() > 0))
|
||||
.max(1)
|
||||
.to_string(),
|
||||
)
|
||||
{
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::RETRY_AFTER, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// 在下游尚未提交时交付 Provider 的 HTTP 失败响应,并应用客户端容量恢复合同
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let capacity_unavailable = matches!(error, EngineError::Provider(error)
|
||||
@@ -233,107 +153,3 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
kind: GatewayErrorKind,
|
||||
) -> (StatusCode, &'static str, &'static str) {
|
||||
match kind {
|
||||
GatewayErrorKind::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
),
|
||||
GatewayErrorKind::Unsupported => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"unsupported_capability",
|
||||
),
|
||||
GatewayErrorKind::Unauthorized => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
),
|
||||
GatewayErrorKind::PolicyDenied => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"invalid_request_error",
|
||||
"policy_denied",
|
||||
),
|
||||
GatewayErrorKind::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
),
|
||||
GatewayErrorKind::NoAvailableProvider => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_full",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"concurrency_queue_timeout",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
"rate_limit_exceeded",
|
||||
),
|
||||
GatewayErrorKind::BudgetExhausted => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"usage_limit_reached",
|
||||
"usage_limit_reached",
|
||||
),
|
||||
GatewayErrorKind::UpstreamUnavailable => (
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
"request_timeout",
|
||||
),
|
||||
GatewayErrorKind::Cancelled => (
|
||||
StatusCode::REQUEST_TIMEOUT,
|
||||
"server_error",
|
||||
"request_cancelled",
|
||||
),
|
||||
GatewayErrorKind::Internal => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
_ => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"server_error",
|
||||
"internal_error",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,34 +5,31 @@ use gateway_admin::model::peer_link::{PeerLinkExport, PeerModelState as ModelSta
|
||||
use gateway_core::metering::Decimal;
|
||||
use gateway_protocol::peer::{
|
||||
OBJECT, PeerBudgetWindow, PeerComponentStatus, PeerKeyUsage, PeerLink, PeerModelAvailability,
|
||||
PeerModelCoverage, PeerModelState, PeerPlanCount, PeerPool, PeerPoolDisclosure, PeerPoolStates,
|
||||
PeerQuota, PeerQuotaAccount, PeerQuotaDisclosure, PeerQuotaWindow, PeerUsage, SCHEMA_VERSION,
|
||||
PeerModelCoverage, PeerModelState, PeerPlanCount, PeerPool, PeerPoolStates, PeerQuota,
|
||||
PeerQuotaAccount, PeerQuotaWindow, PeerUsage, SCHEMA_VERSION,
|
||||
};
|
||||
use std::time::SystemTime;
|
||||
|
||||
pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
let mut capabilities = vec!["key_usage".to_owned()];
|
||||
let pool = export.pool.map_or_else(PeerPool::default, |pool| {
|
||||
let pool = export.pool.map(|pool| {
|
||||
capabilities.push("pool_summary".to_owned());
|
||||
PeerPool {
|
||||
disclosure: PeerPoolDisclosure::Summary,
|
||||
observed_at: Some(export.observed_at.to_rfc3339()),
|
||||
account_count: Some(
|
||||
pool.ready
|
||||
+ pool.cooldown
|
||||
+ pool.exhausted
|
||||
+ pool.disabled
|
||||
+ pool.error
|
||||
+ pool.unknown,
|
||||
),
|
||||
states: Some(PeerPoolStates {
|
||||
account_count: pool.ready
|
||||
+ pool.cooldown
|
||||
+ pool.exhausted
|
||||
+ pool.disabled
|
||||
+ pool.error
|
||||
+ pool.unknown,
|
||||
states: PeerPoolStates {
|
||||
ready: pool.ready,
|
||||
cooldown: pool.cooldown,
|
||||
exhausted: pool.exhausted,
|
||||
disabled: pool.disabled,
|
||||
error: pool.error,
|
||||
unknown: pool.unknown,
|
||||
}),
|
||||
},
|
||||
plans: pool.plans.map(|plans| {
|
||||
plans
|
||||
.into_iter()
|
||||
@@ -41,10 +38,9 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
}),
|
||||
}
|
||||
});
|
||||
let quota = export.quota.map_or_else(PeerQuota::default, |accounts| {
|
||||
let quota = export.quota.map(|accounts| {
|
||||
capabilities.extend(["quota_windows".to_owned(), "model_availability".to_owned()]);
|
||||
PeerQuota {
|
||||
disclosure: PeerQuotaDisclosure::Accounts,
|
||||
accounts: accounts
|
||||
.into_iter()
|
||||
.map(|account| PeerQuotaAccount {
|
||||
@@ -57,8 +53,8 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
.into_iter()
|
||||
.map(|window| PeerQuotaWindow {
|
||||
key: window.key,
|
||||
group: window.group,
|
||||
label: window.label,
|
||||
group: Some(window.group),
|
||||
label: Some(window.label),
|
||||
limit_id: window.limit_id,
|
||||
limit_name: window.limit_name,
|
||||
role: window.role.map(|role| role.as_str().to_owned()),
|
||||
@@ -78,26 +74,34 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
capabilities,
|
||||
observed_at: export.observed_at.to_rfc3339(),
|
||||
expires_at: export.expires_at.to_rfc3339(),
|
||||
usage: PeerUsage {
|
||||
usage: Some(PeerUsage {
|
||||
status: PeerComponentStatus::Available,
|
||||
observed_at: Some(export.budget_observed_at.to_rfc3339()),
|
||||
data: Some(PeerKeyUsage {
|
||||
unit: "USD".to_owned(),
|
||||
unit: Some("USD".to_owned()),
|
||||
allowed: Some(export.budget_access.allowed()),
|
||||
daily: budget_window(
|
||||
export.budget.limits.daily_usd,
|
||||
export.budget.daily_used_usd,
|
||||
export.budget.daily_resets_at,
|
||||
export.budget_access.daily_limit_reached,
|
||||
),
|
||||
weekly: budget_window(
|
||||
export.budget.limits.weekly_usd,
|
||||
export.budget.weekly_used_usd,
|
||||
export.budget.weekly_resets_at,
|
||||
export.budget_access.weekly_limit_reached,
|
||||
),
|
||||
windows: vec![
|
||||
budget_window(
|
||||
"daily",
|
||||
"日额度",
|
||||
86400,
|
||||
export.budget.limits.daily_usd,
|
||||
export.budget.daily_used_usd,
|
||||
export.budget.daily_resets_at,
|
||||
export.budget_access.daily_limit_reached,
|
||||
),
|
||||
budget_window(
|
||||
"weekly",
|
||||
"周额度",
|
||||
604800,
|
||||
export.budget.limits.weekly_usd,
|
||||
export.budget.weekly_used_usd,
|
||||
export.budget.weekly_resets_at,
|
||||
export.budget_access.weekly_limit_reached,
|
||||
),
|
||||
],
|
||||
}),
|
||||
},
|
||||
}),
|
||||
pool,
|
||||
quota,
|
||||
models: export
|
||||
@@ -123,6 +127,9 @@ pub(super) fn response(export: PeerLinkExport) -> PeerLink {
|
||||
}
|
||||
|
||||
fn budget_window(
|
||||
key: &str,
|
||||
label: &str,
|
||||
window_seconds: u64,
|
||||
total: Decimal,
|
||||
used: Decimal,
|
||||
resets_at: Option<SystemTime>,
|
||||
@@ -130,10 +137,13 @@ fn budget_window(
|
||||
) -> PeerBudgetWindow {
|
||||
let limited = total != Decimal::ZERO;
|
||||
PeerBudgetWindow {
|
||||
key: key.to_owned(),
|
||||
label: Some(label.to_owned()),
|
||||
window_seconds: Some(window_seconds),
|
||||
limit_reached: Some(limit_reached),
|
||||
total: limited.then(|| total.canonical()),
|
||||
used: used.canonical(),
|
||||
remaining: limited.then(|| total.saturating_sub(used).canonical()),
|
||||
resets_at: resets_at.map(|time| DateTime::<Utc>::from(time).to_rfc3339()),
|
||||
reset_at: resets_at.map(|time| DateTime::<Utc>::from(time).to_rfc3339()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,39 +1,8 @@
|
||||
//! OpenAI Responses adapter 的稳定错误 contract
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
use thiserror::Error;
|
||||
|
||||
/// OpenAI 风格的协议错误 JSON
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolErrorBody {
|
||||
/// 错误对象
|
||||
pub error: ProtocolError,
|
||||
}
|
||||
|
||||
/// 不包含请求正文或 prompt 的错误对象
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ProtocolError {
|
||||
/// OpenAI 错误类别
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
/// 稳定机器码
|
||||
pub code: &'static str,
|
||||
/// 安全的可读信息
|
||||
pub message: String,
|
||||
/// 出错字段;不包含字段值
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
}
|
||||
|
||||
impl ProtocolErrorBody {
|
||||
/// 序列化为 JSON value
|
||||
#[must_use]
|
||||
pub fn into_value(self) -> Value {
|
||||
serde_json::json!({ "error": self.error })
|
||||
}
|
||||
}
|
||||
|
||||
/// Responses 请求解码错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum RequestDecodeError {
|
||||
|
||||
@@ -943,7 +943,7 @@ impl StreamingExecutionBody {
|
||||
Bytes::from(response_failed_sse_event_with_id(
|
||||
self.encoder.response_id(),
|
||||
error.client_error_type().unwrap_or(default_type),
|
||||
super::super::error::client_error_code(
|
||||
crate::client_error::client_error_code(
|
||||
error.client_error_code().unwrap_or(default_code),
|
||||
),
|
||||
error.client_message(),
|
||||
|
||||
@@ -8,8 +8,9 @@ mod response;
|
||||
mod validation;
|
||||
pub mod websocket;
|
||||
|
||||
pub use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
pub(crate) use crate::request_context::request_client_context;
|
||||
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub use error::{RequestDecodeError, ResponseEncodeError};
|
||||
pub(in crate::openai) use http::{HttpResponseAdapter, execute_prepared_response};
|
||||
pub(crate) use http::{
|
||||
ResponseAuthorization, ResponsesHttpRequest, execute_prepared_responses, responses,
|
||||
|
||||
@@ -10,7 +10,7 @@ use gateway_protocol::openai::sse::{
|
||||
use serde_json::Value;
|
||||
|
||||
use super::error::ResponseEncodeError;
|
||||
use crate::openai::error::capacity_error_for_client;
|
||||
use crate::client_error::capacity_error_for_client;
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Responses WebSocket 入站请求与下行事件的纯协议映射
|
||||
|
||||
use crate::client_error::capacity_error_for_client;
|
||||
|
||||
use axum::http::{HeaderName, HeaderValue, StatusCode};
|
||||
use gateway_core::engine::EngineError;
|
||||
use gateway_core::error::ProviderErrorKind;
|
||||
@@ -7,9 +9,7 @@ use gateway_core::event::ProviderResponseHeader;
|
||||
use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::openai::error::{
|
||||
capacity_error_for_client, gateway_error_contract, gateway_error_from_engine,
|
||||
};
|
||||
use crate::openai::error::{gateway_error_contract, gateway_error_from_engine};
|
||||
|
||||
use super::super::{
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError,
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
//! 管理与自助页面共用的时间展示投影
|
||||
//! 管理与自助页面共用的时间查询解析与展示投影
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::time::DeploymentTimeZone;
|
||||
|
||||
use crate::WireValidationError;
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn query_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
@@ -10,8 +22,7 @@ pub(crate) fn query_range(
|
||||
as_of: Option<i64>,
|
||||
timezone: DeploymentTimeZone,
|
||||
default_period: &str,
|
||||
) -> Result<gateway_admin::model::observability::TimeRange, crate::admin::WireValidationError> {
|
||||
use crate::admin::{WireValidationError, observability::parse_datetime};
|
||||
) -> Result<gateway_admin::model::observability::TimeRange, WireValidationError> {
|
||||
use gateway_admin::model::observability::{CalendarPeriod, TimeRange};
|
||||
if start.is_some() || end.is_some() {
|
||||
if period.is_some() || as_of.is_some() {
|
||||
|
||||
@@ -851,35 +851,21 @@ mod actions {
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"data": {},
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_complete_should_not_accept_group_assignment() {
|
||||
fn oauth_complete_rejects_unknown_fields() {
|
||||
let flow_id = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7_u8; 32]);
|
||||
let request: CompleteAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
let mut payload = json!({
|
||||
"provider": "openai",
|
||||
"flowId": flow_id,
|
||||
"callbackUrl": "http://localhost/callback?code=ok"
|
||||
}))
|
||||
.expect("decode OAuth completion");
|
||||
});
|
||||
let request: CompleteAccountAuthorizationRequest =
|
||||
serde_json::from_value(payload.clone()).expect("decode OAuth completion");
|
||||
assert!(request.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<CompleteAccountAuthorizationRequest>(json!({
|
||||
"provider": "xai",
|
||||
"flowId": "flow_test",
|
||||
"callbackUrl": "http://localhost/callback?code=ok",
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
payload["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<CompleteAccountAuthorizationRequest>(payload).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -651,6 +651,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
max_waiting_per_key: command.values.max_waiting_per_key,
|
||||
max_waiting_per_account: command.values.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.values.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: command.values.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: command
|
||||
.values
|
||||
.openai_guardian_reserved_concurrency,
|
||||
@@ -1893,6 +1894,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -656,6 +656,23 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
);
|
||||
}
|
||||
|
||||
fn peer_summary(
|
||||
discovered: bool,
|
||||
observed: bool,
|
||||
) -> gateway_admin::model::provider_credentials::ProviderQuota {
|
||||
gateway_admin::model::provider_credentials::ProviderQuota {
|
||||
upstream: Some(gateway_admin::model::upstream_info::ProviderUpstreamInfo {
|
||||
discovered,
|
||||
observed_at: observed.then(chrono::Utc::now),
|
||||
stale: true,
|
||||
connection_revision: "test-connection".to_owned(),
|
||||
last_error: None,
|
||||
pool: None,
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
use axum::{
|
||||
@@ -670,13 +687,17 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
for (authentication_kind, plan_type, plan_display) in [
|
||||
(Some("oauth"), Some("pro"), Some("Pro")),
|
||||
(Some("api_key"), None, None),
|
||||
(None, None, None),
|
||||
for (authentication_kind, plan_type, plan_display, peer) in [
|
||||
(Some("oauth"), Some("pro"), Some("Pro"), None),
|
||||
(Some("api_key"), None, None, None),
|
||||
(Some("api_key"), None, None, Some((true, true))),
|
||||
(Some("api_key"), None, None, Some((true, false))),
|
||||
(Some("api_key"), None, None, Some((false, true))),
|
||||
(None, None, None, None),
|
||||
] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
|
||||
*fixture.diagnostics.lock().expect("diagnostics") = DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 4,
|
||||
@@ -718,6 +739,10 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["accountUpstreamPeer"],
|
||||
json!(peer.map(|(d, o)| d && o))
|
||||
);
|
||||
assert_eq!(value["data"]["items"][0]["requestShare"], 0.5);
|
||||
assert_eq!(value["data"]["items"][0]["retryCount"], 3);
|
||||
assert_eq!(value["data"]["items"][0]["retryRate"], 0.5);
|
||||
@@ -879,7 +904,7 @@ async fn usage_route_should_forward_name_and_model_contains_filters() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
async fn usage_route_projects_request_facts_and_billing() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
@@ -906,6 +931,7 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
@@ -1123,7 +1149,6 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
"userAgent": "codex-cli/1.0",
|
||||
})
|
||||
);
|
||||
assert!(value["data"]["items"][0].get("metadata").is_none());
|
||||
assert_eq!(
|
||||
value["data"]["items"][1]["billing"]["totalAmountDisplay"],
|
||||
"≈ $0.007"
|
||||
@@ -1132,6 +1157,39 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
value["data"]["items"][2]["billing"]["totalAmountDisplay"],
|
||||
"$0.007"
|
||||
);
|
||||
|
||||
// 互联标识随当前摘要变化,重复请求共享同一账号,既有请求快照保持原值
|
||||
for peer in [
|
||||
Some((true, true)),
|
||||
Some((true, false)),
|
||||
Some((false, true)),
|
||||
None,
|
||||
] {
|
||||
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?status=all")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_peer_badge")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
for item in value["data"]["items"].as_array().unwrap() {
|
||||
assert_eq!(
|
||||
item["accountUpstreamPeer"],
|
||||
json!(peer.map(|(d, o)| d && o))
|
||||
);
|
||||
assert_eq!(item["accountName"], "Snapshot Alpha");
|
||||
assert_eq!(item["accountPlanType"], "supergrok_plus");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1731,7 +1789,7 @@ async fn usage_status_query_accepts_three_modes_and_rejects_unknown_values() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_identify_metrics_without_localized_labels() {
|
||||
async fn dashboard_summary_identifies_trend_metrics_and_account_usage_values() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
@@ -1884,7 +1942,6 @@ async fn dashboard_summary_should_identify_metrics_without_localized_labels() {
|
||||
assert_eq!(summary.len(), metrics.len());
|
||||
for (item, metric) in summary.iter().zip(metrics) {
|
||||
assert_eq!(item["metric"], metric);
|
||||
assert!(item.get("label").is_none());
|
||||
assert!(item["value"].is_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ fn update_body() -> Value {
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"maxRetryAfterSeconds": 240,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
@@ -426,112 +427,20 @@ fn settings_request_should_reject_non_semver_client_min() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
fn settings_response_formats_update_time_in_deployment_timezone() {
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::account::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
PublicModelId::new("gpt-5.4").expect("public model"),
|
||||
UpstreamModelId::new("gpt-5.5").expect("upstream model"),
|
||||
),
|
||||
(
|
||||
PublicModelId::new("grok-latest").expect("public model"),
|
||||
UpstreamModelId::new("grok-4.5").expect("upstream model"),
|
||||
),
|
||||
]),
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
.expect("timestamp"),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_error_protection: Default::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
};
|
||||
|
||||
let mut settings = super::test_runtime_settings();
|
||||
settings.updated_at = Utc.with_ymd_and_hms(2026, 8, 2, 10, 30, 0).unwrap();
|
||||
let value = serde_json::to_value(RuntimeSettingsView::from((
|
||||
settings,
|
||||
gateway_api::TimePresenter::new(Default::default()),
|
||||
)))
|
||||
.expect("serialize view");
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"configRevision": 7,
|
||||
"providerRequestProfiles": {},
|
||||
"openaiClientProfile": null,
|
||||
"xaiClientProfile": null,
|
||||
"codexPrivacyPolicy": {"enabled":false,"onError":"skip_rule","rules":[]},
|
||||
"requestLocationEnabled": false,
|
||||
"requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"},
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
},
|
||||
"refreshMarginSeconds": 1800,
|
||||
"refreshConcurrency": 4,
|
||||
"maxConcurrentPerAccount": 5,
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"openaiGuardianReservedConcurrency": 0,
|
||||
"openaiAccountAffinity": "relaxed",
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"smartSchedulingDefaults": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 31,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountErrorProtection": gateway_core::account::AccountErrorProtection::default(),
|
||||
"accountWarmupEnabled": false,
|
||||
"accountWarmupScheduleTime": "08:00",
|
||||
"accountWarmupModel": null,
|
||||
"resetCreditLeadMinutes": 10,
|
||||
"resetCreditMaxWeeklyRemainingPercent": null,
|
||||
"updatedAt": "2026-08-02T10:30:00Z",
|
||||
"updatedAtDisplay": "2026-08-02 18:30:00"
|
||||
})
|
||||
);
|
||||
|
||||
assert_eq!(value["updatedAt"], "2026-08-02T10:30:00Z");
|
||||
assert_eq!(value["updatedAtDisplay"], "2026-08-02 18:30:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -92,6 +92,7 @@ pub(super) async fn fixture() -> AdminTestFixture {
|
||||
|
||||
fn usage_record() -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
|
||||
@@ -243,10 +243,10 @@ async fn exports_scoped_observations_even_when_the_key_budget_is_exhausted_witho
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["object"], "cpr.peer");
|
||||
assert_eq!(body["schema_version"], 1);
|
||||
assert_eq!(body["usage"]["data"]["daily"]["remaining"], "0");
|
||||
assert_eq!(body["schema_version"], 2);
|
||||
assert_eq!(body["usage"]["data"]["windows"][0]["remaining"], "0");
|
||||
assert_eq!(body["usage"]["data"]["allowed"], false);
|
||||
assert_eq!(body["usage"]["data"]["daily"]["limit_reached"], true);
|
||||
assert_eq!(body["usage"]["data"]["windows"][0]["limit_reached"], true);
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert!(body["pool"].get("plans").is_none());
|
||||
assert_eq!(
|
||||
@@ -261,6 +261,114 @@ async fn exports_scoped_observations_even_when_the_key_budget_is_exhausted_witho
|
||||
assert!(fixture.observations.lock().unwrap().summaries.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_plans_use_account_plan_fallback_independently_of_quota_sharing() {
|
||||
for share_quota in [true, false] {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
for (account_plan, quota_plan, expected) in [
|
||||
(None, Some("free"), Some("Free")),
|
||||
(Some("unknown"), Some("free"), Some("Free")),
|
||||
(Some(" "), Some("free"), Some("Free")),
|
||||
(Some("edu_pro"), Some("edu"), Some("EduPro")),
|
||||
(Some("free"), None, Some("Free")),
|
||||
(None, None, None),
|
||||
(None, Some("unknown"), None),
|
||||
] {
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = account_plan.map(str::to_owned);
|
||||
fixture
|
||||
.provider_quota
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.plan_type = quota_plan.map(str::to_owned);
|
||||
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
let plans =
|
||||
expected.map_or_else(|| json!([]), |name| json!([{ "name": name, "count": 1 }]));
|
||||
assert_eq!(
|
||||
body["pool"]["plans"], plans,
|
||||
"{account_plan:?} / {quota_plan:?}"
|
||||
);
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body.get("quota").is_some(), share_quota);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type
|
||||
.as_deref(),
|
||||
account_plan,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_only_summary_limits_exported_data_and_preserves_unknown_on_quota_read_failure() {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota: false,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = None;
|
||||
{
|
||||
let mut quota = fixture.provider_quota.lock().unwrap();
|
||||
let quota = quota.as_mut().unwrap();
|
||||
quota.plan_type = Some("free".into());
|
||||
quota.windows =
|
||||
vec![quota.windows[0].clone(); gateway_protocol::peer::MAX_QUOTA_WINDOWS + 1];
|
||||
}
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(
|
||||
body["pool"]["plans"],
|
||||
json!([{ "name": "Free", "count": 1 }])
|
||||
);
|
||||
assert_eq!(body["capabilities"], json!(["key_usage", "pool_summary"]));
|
||||
assert!(body.get("quota").is_none());
|
||||
|
||||
*fixture.provider_quota.lock().unwrap() = None;
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body["pool"]["plans"], json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requires_authentication_explicit_sharing_and_bound_groups_and_rejects_query_overrides() {
|
||||
let (fixture, app) = fixture(sharing(), RAW_KEY).await;
|
||||
@@ -338,7 +446,7 @@ async fn current_field_permissions_and_direct_account_facts_narrow_the_frozen_sc
|
||||
.peer_sharing
|
||||
.share_quota = false;
|
||||
let body = response_json(query(&app, PATH, Some(RAW_KEY)).await).await;
|
||||
assert_eq!(body["quota"], json!({ "disclosure": "none" }));
|
||||
assert!(body.get("quota").is_none());
|
||||
assert!(body.get("models").is_none());
|
||||
fixture
|
||||
.account
|
||||
@@ -511,7 +619,7 @@ async fn heavy_snapshots_reject_excess_concurrency_and_release_slots_on_completi
|
||||
.await
|
||||
.expect("budget-only queries do not consume heavy-query slots");
|
||||
assert_eq!(budget.status(), StatusCode::OK);
|
||||
assert_eq!(response_json(budget).await["pool"]["disclosure"], "none");
|
||||
assert!(response_json(budget).await.get("pool").is_none());
|
||||
let cancelled = requests.pop().unwrap();
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.unwrap_err().is_cancelled());
|
||||
|
||||
@@ -1677,6 +1677,10 @@ where
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
// 服务器建议不可截短;超出请求冻结的上限后沿用安全换号或原错误出口
|
||||
let retry_after_exceeds_limit = error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after());
|
||||
// Provider 的有界同账号传输恢复是尽力重试策略,不要求把 Sent/Ambiguous
|
||||
// 改写成安全证明;外部副作用、取消、预算与下游提交仍由 Core 拦截
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
@@ -1699,7 +1703,8 @@ where
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if !execution_effect_observed
|
||||
}) if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1707,7 +1712,8 @@ where
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if !execution_effect_observed
|
||||
if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1753,7 +1759,7 @@ where
|
||||
max_delay,
|
||||
jitter,
|
||||
..
|
||||
}) if ordinary_retry && !suppress_transient_retry => {
|
||||
}) if ordinary_retry && !suppress_transient_retry && !retry_after_exceeds_limit => {
|
||||
let retries = self
|
||||
.transient_retry_counts
|
||||
.entry(current.metadata.provider_account_id().clone())
|
||||
@@ -1832,6 +1838,8 @@ where
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"retryAfterExceeded": retry_after_exceeds_limit,
|
||||
"maxRetryAfterSeconds": self.plan.max_retry_after().as_secs(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
"connectionRetry": connection_retry_requested,
|
||||
|
||||
@@ -576,6 +576,7 @@ pub struct RoutingPlan {
|
||||
account_error_protection: Arc<crate::account::CompiledAccountErrorProtection>,
|
||||
account_error_protection_revision: u64,
|
||||
request_body_limit: usize,
|
||||
max_retry_after: std::time::Duration,
|
||||
pricing: Arc<crate::metering::PricingOverrides>,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
privacy: Option<Arc<dyn crate::settings::privacy::CompiledPrivacyPolicy>>,
|
||||
@@ -598,6 +599,12 @@ impl RoutingPlan {
|
||||
Arc::clone(&self.account_error_protection)
|
||||
}
|
||||
|
||||
/// 本次请求接受的单次 Retry-After 等待上限
|
||||
#[must_use]
|
||||
pub const fn max_retry_after(&self) -> std::time::Duration {
|
||||
self.max_retry_after
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> usize {
|
||||
self.request_body_limit
|
||||
|
||||
@@ -1408,6 +1408,9 @@ impl RuntimeSnapshot {
|
||||
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
@@ -1495,6 +1498,9 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
|
||||
@@ -21,6 +21,7 @@ pub struct SettingsValues {
|
||||
pub(crate) max_waiting_per_key: u32,
|
||||
pub(crate) max_waiting_per_account: u32,
|
||||
pub(crate) concurrency_wait_timeout_seconds: u32,
|
||||
pub(crate) max_retry_after_seconds: u32,
|
||||
pub(crate) openai_guardian_reserved_concurrency: u32,
|
||||
pub(crate) openai_account_affinity: crate::account::AccountAffinity,
|
||||
pub(crate) max_account_rotations: u32,
|
||||
@@ -84,6 +85,13 @@ impl SettingsValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// 网关接受的单次服务器建议等待上限,超过时交由既有换号或错误出口处理
|
||||
#[must_use]
|
||||
pub const fn with_max_retry_after_seconds(mut self, seconds: u32) -> Self {
|
||||
self.max_retry_after_seconds = seconds;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
|
||||
self.max_account_rotations = rotations;
|
||||
@@ -178,6 +186,7 @@ impl SettingsValues {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: crate::account::AccountAffinity::default(),
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -1174,6 +1174,7 @@ fn plan_with_profiles(
|
||||
fast_mode,
|
||||
profiles,
|
||||
Default::default(),
|
||||
240,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1184,6 +1185,7 @@ fn plan_with_protection(
|
||||
fast_mode: FastMode,
|
||||
profiles: BTreeMap<ProviderKind, gateway_core::account::OpaqueProviderData>,
|
||||
protection: gateway_core::account::AccountErrorProtection,
|
||||
max_retry_after_seconds: u32,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
@@ -1237,7 +1239,8 @@ fn plan_with_protection(
|
||||
.unwrap(),
|
||||
)
|
||||
.with_request_location(request_location, true)
|
||||
.with_account_error_protection(protection),
|
||||
.with_account_error_protection(protection)
|
||||
.with_max_retry_after_seconds(max_retry_after_seconds),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -4216,6 +4219,177 @@ fn server_retry_advice_should_override_local_backoff_and_transport_fallback() {
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_after_plan(operation: &Operation, limit: u32) -> RoutingPlan {
|
||||
plan_with_protection(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::MIN),
|
||||
Default::default(),
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
limit,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_at_or_below_the_configured_limit_keeps_the_full_wait() {
|
||||
for (limit, seconds) in [(240, 240), (60, 60), (400, 308)] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transient_rejection().with_retry_after(Duration::from_secs(seconds))
|
||||
)],
|
||||
}]);
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(600)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
cancellation.clone(),
|
||||
))
|
||||
.unwrap();
|
||||
let trace = session.trace().clone();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(block_on(async { futures::poll!(&mut collection) }).is_pending());
|
||||
let trace = trace.snapshot().unwrap();
|
||||
let retry = trace["events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "retry.decided")
|
||||
.unwrap();
|
||||
assert_eq!(retry["data"]["delayMs"], seconds * 1000);
|
||||
assert_eq!(retry["data"]["retryAfterExceeded"], false);
|
||||
cancellation.cancel();
|
||||
assert!(matches!(block_on(collection), Err(EngineError::Cancelled)));
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_above_the_configured_limit_rotates_without_waiting() {
|
||||
for (limit, seconds) in [(240, 241), (240, 308), (60, 61), (0, 1)] {
|
||||
for rejection in [
|
||||
transient_rejection(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_fallback(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(seconds)),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection.with_retry_after(Duration::from_secs(seconds)))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(matches!(
|
||||
block_on(async { futures::poll!(&mut collection) }),
|
||||
std::task::Poll::Ready(Ok(_))
|
||||
));
|
||||
drop(collection);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(contexts[1].required_account(), None);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excessive_retry_after_preserves_the_error_when_rotation_is_unavailable_or_disallowed() {
|
||||
for (rejection, rotation_allowed) in [
|
||||
(transient_rejection(), true),
|
||||
(
|
||||
transient_rejection().with_transient_retry(
|
||||
NonZeroU32::MIN,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
),
|
||||
false,
|
||||
),
|
||||
(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(308)),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let original = Bytes::from_static(
|
||||
br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"308"}}}"#,
|
||||
);
|
||||
let expected_kind = rejection.kind();
|
||||
let rejection = rejection
|
||||
.with_retry_after(Duration::from_secs(308))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/json".to_vec()),
|
||||
original.clone(),
|
||||
));
|
||||
let mut scripts = vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection)],
|
||||
}];
|
||||
if rotation_allowed {
|
||||
scripts.push(Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)));
|
||||
}
|
||||
let (coordinator, _, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let result = block_on(async { futures::poll!(&mut collection) });
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) = result else {
|
||||
panic!("must return the original failure without waiting")
|
||||
};
|
||||
assert_eq!(error.kind(), expected_kind);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(308)));
|
||||
assert_eq!(
|
||||
error.client_visible_upstream_response().unwrap().body(),
|
||||
&original
|
||||
);
|
||||
assert_eq!(
|
||||
provider.contexts.lock().unwrap().len(),
|
||||
if rotation_allowed { 2 } else { 1 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
for policy in [
|
||||
@@ -4223,7 +4397,7 @@ fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let route_plan = retry_after_plan(&operation, 0);
|
||||
let (coordinator, _, _) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
@@ -4944,7 +5118,9 @@ fn structural_event_before_replay_safe_failure_should_switch_account_before_comm
|
||||
|
||||
#[test]
|
||||
fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery() {
|
||||
for close_window in [false, true] {
|
||||
for (close_window, excessive_retry_after) in
|
||||
[(false, false), (true, false), (false, true), (true, true)]
|
||||
{
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let prelude = || {
|
||||
@@ -4971,11 +5147,12 @@ fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery(
|
||||
// 后续前导不能撤销已经交付未知事件的事实
|
||||
items.push(Ok(prelude()));
|
||||
}
|
||||
items.push(Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe()));
|
||||
items.push(Err(if excessive_retry_after {
|
||||
transient_rejection().with_retry_after(Duration::from_secs(308))
|
||||
} else {
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
|
||||
.with_replay_safe()
|
||||
}));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
@@ -7185,6 +7362,7 @@ fn error_protection_tracks_real_retry_attempts_and_skips_diagnostic_runs() {
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
policy().as_ref().clone(),
|
||||
240,
|
||||
);
|
||||
let feedback = Arc::new(Feedback::default());
|
||||
let failure = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use crate::system_update::SystemUpdateConfig;
|
||||
use crate::{ConfigError, system_update::SystemUpdateConfig};
|
||||
|
||||
const CONFIG_RELATIVE_PATH: &str = "deploy/config.yaml";
|
||||
const SERVER_HOST_ENV: &str = "CPR_SERVER_HOST";
|
||||
@@ -216,26 +216,6 @@ pub struct FileLoggingConfig {
|
||||
pub max_file_size_mb: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("配置文件 {path} 的字段 {field:?} 不合法")]
|
||||
InvalidValue { path: PathBuf, field: String },
|
||||
#[error("current directory is unavailable")]
|
||||
CurrentDirectory,
|
||||
#[error("deploy/config.yaml was not found")]
|
||||
ConfigFileNotFound,
|
||||
#[error("configuration path is invalid")]
|
||||
InvalidConfigPath,
|
||||
#[error("configuration document is invalid: {path}")]
|
||||
InvalidDocument { path: PathBuf },
|
||||
#[error("配置文件 {path} 缺少必填字段 {field:?}")]
|
||||
MissingField { path: PathBuf, field: String },
|
||||
#[error("configuration field is invalid: {0}")]
|
||||
InvalidField(&'static str),
|
||||
#[error("environment variable is invalid: {0}")]
|
||||
InvalidEnvironment(&'static str),
|
||||
}
|
||||
|
||||
fn optional_environment_value(name: &'static str) -> Result<Option<String>, ConfigError> {
|
||||
match env::var(name) {
|
||||
Ok(value) if value.trim().is_empty() => Err(ConfigError::InvalidEnvironment(name)),
|
||||
|
||||
@@ -28,7 +28,34 @@ use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::lifecycle::ConnectionLifecycle;
|
||||
use gateway_core::task::{WorkerContribution, WorkerLeaderLeasePort};
|
||||
|
||||
pub use config::{ConfigError, HostConfig, LoadableConfig, load_config};
|
||||
pub use config::{HostConfig, LoadableConfig, load_config};
|
||||
|
||||
/// Host 配置加载与各能力配置校验共用的脱敏错误
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("配置文件 {path} 的字段 {field:?} 不合法")]
|
||||
InvalidValue {
|
||||
path: std::path::PathBuf,
|
||||
field: String,
|
||||
},
|
||||
#[error("current directory is unavailable")]
|
||||
CurrentDirectory,
|
||||
#[error("deploy/config.yaml was not found")]
|
||||
ConfigFileNotFound,
|
||||
#[error("configuration path is invalid")]
|
||||
InvalidConfigPath,
|
||||
#[error("configuration document is invalid: {path}")]
|
||||
InvalidDocument { path: std::path::PathBuf },
|
||||
#[error("配置文件 {path} 缺少必填字段 {field:?}")]
|
||||
MissingField {
|
||||
path: std::path::PathBuf,
|
||||
field: String,
|
||||
},
|
||||
#[error("configuration field is invalid: {0}")]
|
||||
InvalidField(&'static str),
|
||||
#[error("environment variable is invalid: {0}")]
|
||||
InvalidEnvironment(&'static str),
|
||||
}
|
||||
|
||||
use self::client_distribution::RgAdguardClientDistribution;
|
||||
use self::logging::{LogGuard, initialize_logging};
|
||||
|
||||
@@ -46,7 +46,7 @@ pub enum LogError {
|
||||
#[error("logging size limit is too large")]
|
||||
SizeOverflow,
|
||||
#[error(transparent)]
|
||||
InvalidConfig(#[from] crate::config::ConfigError),
|
||||
InvalidConfig(#[from] crate::ConfigError),
|
||||
}
|
||||
|
||||
/// 初始化日志;按各通道的保留窗口清理,不以分片数量淘汰记录
|
||||
|
||||
@@ -6,7 +6,11 @@ use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use super::{OperationError, internal, invalid};
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const APP_BINARY_NAME: &str = if cfg!(windows) {
|
||||
"codex-proxy-rs.exe"
|
||||
} else {
|
||||
"codex-proxy-rs"
|
||||
};
|
||||
const MAX_EXTRACTED_SIZE: u64 = 1024 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_FILES: usize = 20_000;
|
||||
|
||||
@@ -126,14 +130,42 @@ pub(crate) fn extract_release(
|
||||
|
||||
fn unsafe_archive_path(path: &Path) -> bool {
|
||||
path.is_absolute()
|
||||
|| path.components().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::ParentDir | Component::Prefix(_) | Component::RootDir
|
||||
)
|
||||
|| path.components().any(|component| match component {
|
||||
Component::ParentDir | Component::Prefix(_) | Component::RootDir => true,
|
||||
#[cfg(windows)]
|
||||
Component::Normal(name) => unsafe_windows_name(name),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn unsafe_windows_name(name: &std::ffi::OsStr) -> bool {
|
||||
let name = name.to_string_lossy();
|
||||
// 防止归档路径被 Win32 解释为设备、NTFS 备用数据流或另一个规范化文件名
|
||||
if name.ends_with(['.', ' '])
|
||||
|| name
|
||||
.chars()
|
||||
.any(|value| value < ' ' || "<>:\"|?*".contains(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let stem = name
|
||||
.split('.')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_uppercase();
|
||||
matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
|
||||
|| stem
|
||||
.strip_prefix("COM")
|
||||
.or_else(|| stem.strip_prefix("LPT"))
|
||||
.is_some_and(|suffix| {
|
||||
matches!(
|
||||
suffix,
|
||||
"1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn web_dist_relative_path(path: &Path) -> Option<PathBuf> {
|
||||
let components = path
|
||||
.components()
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::{
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{OperationError, conflict, internal};
|
||||
use crate::config::ConfigError;
|
||||
use crate::ConfigError;
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const DEFAULT_GITHUB_API_BASE: &str = "https://api.github.com/repos";
|
||||
|
||||
@@ -16,6 +16,7 @@ mod swap_paths;
|
||||
|
||||
pub use config::SystemUpdateConfig;
|
||||
pub use operations::ProcessSystemOperations;
|
||||
pub use process::startup_arguments;
|
||||
pub use release::validate_download_url;
|
||||
|
||||
use error::{OperationError, conflict, internal, invalid, upstream};
|
||||
|
||||
@@ -41,12 +41,18 @@ pub struct ProcessSystemOperations {
|
||||
cancellation: CancellationToken,
|
||||
events: Arc<UpdateEvents>,
|
||||
config: Arc<SystemUpdateConfig>,
|
||||
operation_lock: Arc<AsyncMutex<Option<OperationFileLock>>>,
|
||||
operation_lock: Arc<AsyncMutex<Option<RestartGuard>>>,
|
||||
release_cache: Arc<ReleaseCache>,
|
||||
running_files: Arc<Result<ReleaseFiles, OperationError>>,
|
||||
diagnostics: super::diagnostics::Diagnostics,
|
||||
}
|
||||
|
||||
struct RestartGuard {
|
||||
// 字段按声明顺序释放,先移除更新锁,再通知替换进程开始启动
|
||||
_file_lock: OperationFileLock,
|
||||
_replacement_input: Option<std::process::ChildStdin>,
|
||||
}
|
||||
|
||||
impl ProcessSystemOperations {
|
||||
#[must_use]
|
||||
pub fn new(cancellation: CancellationToken, mut config: SystemUpdateConfig) -> Self {
|
||||
@@ -75,7 +81,7 @@ impl ProcessSystemOperations {
|
||||
self.diagnostics.bind(diagnostics);
|
||||
}
|
||||
|
||||
fn lock_operation(&self) -> Result<OwnedMutexGuard<Option<OperationFileLock>>, OperationError> {
|
||||
fn lock_operation(&self) -> Result<OwnedMutexGuard<Option<RestartGuard>>, OperationError> {
|
||||
let operation = Arc::clone(&self.operation_lock)
|
||||
.try_lock_owned()
|
||||
.map_err(|_| conflict("已有系统操作正在执行"))?;
|
||||
@@ -559,15 +565,17 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
{
|
||||
return Err(conflict("重启检查期间安装文件已变化,请重新确认"));
|
||||
}
|
||||
let message = if self.config.deployment_mode == "docker" {
|
||||
"已安排进程内重启"
|
||||
let (message, replacement_input) = if self.config.deployment_mode == "docker" {
|
||||
("已安排进程内重启", None)
|
||||
} else {
|
||||
spawn_replacement(&self.config)?;
|
||||
"已安排自重启"
|
||||
("已安排自重启", spawn_replacement(&self.config)?)
|
||||
};
|
||||
let operation_id = operation_id("restart");
|
||||
// 受理后直到旧服务释放,保留重启状态和跨进程锁;延迟关闭只用于送达响应
|
||||
*operation = Some(file_lock);
|
||||
*operation = Some(RestartGuard {
|
||||
_file_lock: file_lock,
|
||||
_replacement_input: replacement_input,
|
||||
});
|
||||
let cancellation = self.cancellation.clone();
|
||||
drop(tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
@@ -2,15 +2,37 @@
|
||||
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::io;
|
||||
use std::process::{ChildStdin, Command, Stdio};
|
||||
|
||||
#[cfg(unix)]
|
||||
use super::config::environment_value;
|
||||
use super::{OperationError, SystemUpdateConfig, internal};
|
||||
|
||||
#[cfg(unix)]
|
||||
const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS";
|
||||
#[cfg(unix)]
|
||||
const DEFAULT_RESTART_DELAY_MS: u64 = 1_200;
|
||||
const RESTART_HANDOFF_ARGUMENT: &str = "--restart-handoff";
|
||||
|
||||
pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), OperationError> {
|
||||
/// 组合根在创建运行时前接收重启交接,并取得不含内部交接标记的 CLI 参数
|
||||
pub fn startup_arguments() -> Result<impl Iterator<Item = std::ffi::OsString>, OperationError> {
|
||||
let mut arguments = env::args_os().skip(1).peekable();
|
||||
if arguments
|
||||
.peek()
|
||||
.is_some_and(|value| value == RESTART_HANDOFF_ARGUMENT)
|
||||
{
|
||||
arguments.next();
|
||||
// 旧服务释放操作锁后关闭管道;旧进程异常退出也会关闭管道,避免固定延迟抢占端口
|
||||
io::copy(&mut io::stdin().lock(), &mut io::sink())
|
||||
.map_err(|error| internal("无法等待旧服务释放资源").with_source(error))?;
|
||||
}
|
||||
Ok(arguments)
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_replacement(
|
||||
config: &SystemUpdateConfig,
|
||||
) -> Result<Option<ChildStdin>, OperationError> {
|
||||
let executable = config.executable_path()?;
|
||||
let metadata = fs::metadata(&executable)
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))?;
|
||||
@@ -25,12 +47,12 @@ pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), Opera
|
||||
}
|
||||
}
|
||||
|
||||
let delay_ms = environment_value(RESTART_DELAY_ENV)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_RESTART_DELAY_MS);
|
||||
#[cfg(unix)]
|
||||
let mut command = {
|
||||
let delay_ms = environment_value(RESTART_DELAY_ENV)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_RESTART_DELAY_MS);
|
||||
let delay = format!("{}.{:03}", delay_ms / 1_000, delay_ms % 1_000);
|
||||
let mut command = Command::new("/bin/sh");
|
||||
command
|
||||
@@ -39,20 +61,26 @@ pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), Opera
|
||||
.arg("codex-proxy-rs-restart")
|
||||
.arg(delay)
|
||||
.arg(&executable)
|
||||
.args(env::args_os().skip(1));
|
||||
.args(env::args_os().skip(1))
|
||||
.stdin(Stdio::null());
|
||||
command
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let mut command = {
|
||||
let mut command = Command::new(&executable);
|
||||
command
|
||||
.args(env::args_os().skip(1))
|
||||
.env(RESTART_DELAY_ENV, delay_ms.to_string());
|
||||
.arg(RESTART_HANDOFF_ARGUMENT)
|
||||
// 连续重启时移除当前进程已消费的交接标记,保留原始参数及编码
|
||||
.args(
|
||||
env::args_os()
|
||||
.skip(1)
|
||||
.skip_while(|value| value == RESTART_HANDOFF_ARGUMENT),
|
||||
)
|
||||
.stdin(Stdio::piped());
|
||||
command
|
||||
};
|
||||
command.stdin(Stdio::null());
|
||||
command
|
||||
.spawn()
|
||||
.map(|_| ())
|
||||
.map(|mut child| child.stdin.take())
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl UpdateTempDir {
|
||||
|
||||
impl Drop for UpdateTempDir {
|
||||
fn drop(&mut self) {
|
||||
if let Err(error) = fs::remove_dir_all(&self.path)
|
||||
if let Err(error) = super::swap::remove_dir_all(&self.path)
|
||||
&& error.kind() != io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!(path = %self.path.display(), error = %error, "清理系统更新临时目录失败");
|
||||
|
||||
@@ -294,12 +294,40 @@ fn protect_official_plugins(_path: &Path) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_dir_all(path: &Path) -> io::Result<()> {
|
||||
pub(crate) fn remove_dir_all(path: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
make_directories_writable(path)?;
|
||||
#[cfg(windows)]
|
||||
make_files_writable(path)?;
|
||||
fs::remove_dir_all(path)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn make_files_writable(path: &Path) -> io::Result<()> {
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
if metadata.file_type().is_symlink() || (!metadata.is_dir() && !metadata.is_file()) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"release directory contains an unsupported file type",
|
||||
));
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
for entry in fs::read_dir(path)? {
|
||||
make_files_writable(&entry?.path())?;
|
||||
}
|
||||
} else if metadata.permissions().readonly() {
|
||||
// tar 的 0444 会成为 Windows 只读属性,清理受管备份前必须移除此属性
|
||||
let mut permissions = metadata.permissions();
|
||||
#[expect(
|
||||
clippy::permissions_set_readonly_false,
|
||||
reason = "仅 Windows 清理只读属性,不会放宽 Unix 权限"
|
||||
)]
|
||||
permissions.set_readonly(false);
|
||||
fs::set_permissions(path, permissions)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn make_directories_writable(path: &Path) -> io::Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
@@ -53,7 +53,7 @@ fn host_config_should_reject_empty_update_asset_directory() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
fn host_config_resolves_resource_paths_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
@@ -66,34 +66,10 @@ fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
config.logging.file.directory,
|
||||
PathBuf::from("/srv/gateway/.runtime/logs")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
config.runtime_data_dir(),
|
||||
PathBuf::from("/srv/gateway/runtime-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_derives_update_paths_from_runtime_data_dir() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
config.system_update.update_state_file,
|
||||
@@ -124,23 +100,19 @@ fn host_config_rejects_zero_worker_shutdown_window() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_rejects_zero_retention_windows() {
|
||||
for request_dump in [false, true] {
|
||||
let mut config = valid_config();
|
||||
if request_dump {
|
||||
config.logging.request_dump_retention_days = 0.0;
|
||||
} else {
|
||||
config.logging.file.retention_days = 0;
|
||||
}
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
fn file_logging_rejects_zero_retention() {
|
||||
let mut config = valid_config();
|
||||
config.logging.file.retention_days = 0;
|
||||
let error = config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_host::ConfigError::InvalidField("host.logging.file.retention_days")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -6,13 +6,25 @@ use std::{
|
||||
};
|
||||
|
||||
fn main() {
|
||||
let executable = std::env::current_exe().unwrap();
|
||||
let directory = executable.parent().unwrap();
|
||||
if directory.join("restart-probe").is_file() {
|
||||
std::fs::write(directory.join("replacement-waiting"), b"ready").unwrap();
|
||||
let arguments = gateway_host::system_update::startup_arguments()
|
||||
.unwrap()
|
||||
.collect::<Vec<_>>();
|
||||
assert!(!arguments.iter().any(|value| value == "--restart-handoff"));
|
||||
assert!(!directory.join("update.lock").exists());
|
||||
std::fs::write(directory.join("replacement-ran"), b"started").unwrap();
|
||||
return;
|
||||
}
|
||||
let mode = std::fs::read_to_string("mode").unwrap();
|
||||
if mode == "stderr" {
|
||||
std::io::stderr().write_all(&[b'x'; 4096]).unwrap();
|
||||
}
|
||||
if matches!(
|
||||
mode.as_str(),
|
||||
"busy" | "stdin" | "stdout" | "exit" | "allocate"
|
||||
"busy" | "stdin" | "stdout" | "exit" | "allocate" | "update-probe"
|
||||
) {
|
||||
std::io::stdout().write_all(b"R").unwrap();
|
||||
std::io::stdout().flush().unwrap();
|
||||
@@ -25,6 +37,11 @@ fn main() {
|
||||
std::io::stdin().read_exact(&mut [0; 1]).unwrap();
|
||||
std::fs::write("unblocked", b"stdin").unwrap();
|
||||
}
|
||||
"update-probe" => {
|
||||
std::io::stdin().read_exact(&mut [0; 1]).unwrap();
|
||||
std::fs::write("unblocked", b"still-running").unwrap();
|
||||
return;
|
||||
}
|
||||
"stdout" => {
|
||||
// 测试端不消费此正文,超过管道容量后必须由监督器终止,而不是等待写入完成
|
||||
std::io::stdout()
|
||||
|
||||
@@ -18,8 +18,7 @@ async fn manual_image_replacement_should_clear_pending_state_and_invalid_backup(
|
||||
assert!(complete_update(&service, TARGET_VERSION).await.need_restart);
|
||||
// 模拟重建镜像:数据卷保留,应用文件和容器内备份被重新替换
|
||||
fixture.write_executable("manually-installed-binary");
|
||||
fs::remove_file(fixture.root.path().join("codex-proxy-rs.backup"))
|
||||
.expect("remove image backup");
|
||||
fs::remove_file(fixture.backup()).expect("remove image backup");
|
||||
let mut config = fixture.config(&format!("{}/repos", server.uri()));
|
||||
config.version = "1.1.0".into();
|
||||
config.deployment_mode = "docker".into();
|
||||
@@ -126,11 +125,7 @@ async fn pending_update_should_revoke_rollback_when_backup_content_changes() {
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
assert!(complete_update(&service, TARGET_VERSION).await.need_restart);
|
||||
fs::write(
|
||||
fixture.root.path().join("codex-proxy-rs.backup"),
|
||||
"unrelated-backup",
|
||||
)
|
||||
.expect("changed backup");
|
||||
fs::write(fixture.backup(), "unrelated-backup").expect("changed backup");
|
||||
let status = service
|
||||
.update_status()
|
||||
.await
|
||||
|
||||
@@ -40,6 +40,12 @@ use tar::{Builder, EntryType, Header};
|
||||
use wiremock::matchers::{method, path, query_param};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
const BINARY_NAME: &str = if cfg!(windows) {
|
||||
"codex-proxy-rs.exe"
|
||||
} else {
|
||||
"codex-proxy-rs"
|
||||
};
|
||||
|
||||
const OLD_RELEASE_MANIFEST: &str = r#"{"gateway_version":"1.0.0"}"#;
|
||||
|
||||
const TARGET_VERSION: &str = "1.9.9";
|
||||
@@ -253,7 +259,11 @@ impl Fixture {
|
||||
}
|
||||
|
||||
fn executable(&self) -> PathBuf {
|
||||
self.root.path().join("codex-proxy-rs")
|
||||
self.root.path().join(BINARY_NAME)
|
||||
}
|
||||
|
||||
fn backup(&self) -> PathBuf {
|
||||
self.root.path().join(format!("{BINARY_NAME}.backup"))
|
||||
}
|
||||
|
||||
fn web(&self) -> PathBuf {
|
||||
@@ -393,7 +403,7 @@ impl Fixture {
|
||||
#[derive(Clone, Copy)]
|
||||
enum ArchiveKind {
|
||||
Safe,
|
||||
UnsafePath,
|
||||
UnsafePath(&'static str),
|
||||
MissingOfficialManifest,
|
||||
NestedOfficialAsset,
|
||||
OfficialSymlink,
|
||||
@@ -409,7 +419,7 @@ enum ChecksumKind {
|
||||
fn release_archive(kind: ArchiveKind) -> Vec<u8> {
|
||||
let encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
let mut tar = Builder::new(encoder);
|
||||
append_file(&mut tar, "codex-proxy-rs", b"new-binary", false);
|
||||
append_file(&mut tar, BINARY_NAME, b"new-binary", false);
|
||||
append_file(&mut tar, "web/dist/index.html", b"new-web", false);
|
||||
if !matches!(kind, ArchiveKind::MissingOfficialManifest) {
|
||||
append_file(
|
||||
@@ -425,8 +435,8 @@ fn release_archive(kind: ArchiveKind) -> Vec<u8> {
|
||||
b"new-plugin",
|
||||
false,
|
||||
);
|
||||
if matches!(kind, ArchiveKind::UnsafePath) {
|
||||
append_file(&mut tar, "safe", b"escape", true);
|
||||
if let ArchiveKind::UnsafePath(name) = kind {
|
||||
append_file(&mut tar, name, b"escape", true);
|
||||
}
|
||||
if matches!(kind, ArchiveKind::NestedOfficialAsset) {
|
||||
append_file(
|
||||
@@ -461,13 +471,19 @@ fn append_symlink(tar: &mut Builder<GzEncoder<Vec<u8>>>, name: &str, target: &st
|
||||
fn append_file(tar: &mut Builder<GzEncoder<Vec<u8>>>, name: &str, data: &[u8], unsafe_path: bool) {
|
||||
let mut header = Header::new_gnu();
|
||||
header.set_entry_type(EntryType::Regular);
|
||||
header.set_mode(0o755);
|
||||
header.set_mode(if name.starts_with("plugins/official/") {
|
||||
0o444
|
||||
} else {
|
||||
0o755
|
||||
});
|
||||
header.set_size(u64::try_from(data.len()).expect("size"));
|
||||
header.set_path(name).expect("path");
|
||||
header
|
||||
.set_path(if unsafe_path { "safe" } else { name })
|
||||
.expect("path");
|
||||
if unsafe_path {
|
||||
let bytes = header.as_mut_bytes();
|
||||
bytes[..100].fill(0);
|
||||
bytes[..9].copy_from_slice(b"../escape");
|
||||
bytes[..name.len()].copy_from_slice(name.as_bytes());
|
||||
}
|
||||
header.set_cksum();
|
||||
tar.append(&header, data).expect("append");
|
||||
|
||||
@@ -2,6 +2,120 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_and_rollback_preserve_a_running_executable() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fs::copy(
|
||||
env!("CARGO_BIN_EXE_gateway-host-process-worker"),
|
||||
fixture.executable(),
|
||||
)
|
||||
.expect("running executable");
|
||||
let original = fs::read(fixture.executable()).expect("original executable");
|
||||
fs::write(fixture.root.path().join("mode"), b"update-probe").expect("worker mode");
|
||||
let mut child = tokio::process::Command::new(fixture.executable())
|
||||
.current_dir(fixture.root.path())
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.expect("start old executable");
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
child.stdout.as_mut().unwrap().read_u8()
|
||||
)
|
||||
.await
|
||||
.expect("worker ready")
|
||||
.expect("ready byte"),
|
||||
b'R'
|
||||
);
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let status = complete_update(&service, TARGET_VERSION).await;
|
||||
assert_eq!(
|
||||
status.operation.status,
|
||||
SystemOperationStatus::Succeeded,
|
||||
"{status:?}"
|
||||
);
|
||||
assert!(status.need_restart);
|
||||
assert_eq!(fs::read(fixture.backup()).expect("backup"), original);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("installed"),
|
||||
b"new-binary"
|
||||
);
|
||||
assert!(child.try_wait().expect("old process state").is_none());
|
||||
|
||||
service
|
||||
.rollback(Arc::new(AllowingUpdatePreflight))
|
||||
.await
|
||||
.expect("rollback running executable");
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("restored executable"),
|
||||
original
|
||||
);
|
||||
assert!(
|
||||
!service
|
||||
.update_status()
|
||||
.await
|
||||
.expect("restored status")
|
||||
.need_restart
|
||||
);
|
||||
|
||||
service
|
||||
.perform_update(
|
||||
Some(TARGET_VERSION.to_owned()),
|
||||
None,
|
||||
Arc::new(ChangingRevisionPreflight {
|
||||
confirmations: AtomicUsize::new(0),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("second update accepted");
|
||||
let failed = wait_for_update(&service).await;
|
||||
assert_eq!(
|
||||
failed.operation.status,
|
||||
SystemOperationStatus::Failed,
|
||||
"{failed:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("restored after failure"),
|
||||
original
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.web().join("index.html")).expect("restored web"),
|
||||
b"old-web"
|
||||
);
|
||||
|
||||
child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_u8(b'X')
|
||||
.await
|
||||
.expect("release old process");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(5), child.wait())
|
||||
.await
|
||||
.expect("worker exited")
|
||||
.expect("exit status")
|
||||
.success()
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.root.path().join("unblocked")).expect("old process result"),
|
||||
b"still-running"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_not_shutdown_when_replacement_spawn_fails() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -149,7 +263,12 @@ async fn assert_system_operations_rejected(service: &ProcessSystemOperations, re
|
||||
async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
let fixture = Fixture::new();
|
||||
let replacement_marker = fixture.root.path().join("replacement-ran");
|
||||
fixture.write_executable("#!/bin/sh\n: > \"${0%/*}/replacement-ran\"\n");
|
||||
fs::copy(
|
||||
env!("CARGO_BIN_EXE_gateway-host-process-worker"),
|
||||
fixture.executable(),
|
||||
)
|
||||
.expect("replacement executable");
|
||||
fs::write(fixture.root.path().join("restart-probe"), b"").expect("restart probe");
|
||||
let mut config = fixture.config("http://127.0.0.1:1/repos");
|
||||
config.self_restart_enabled = true;
|
||||
let shutdown = CancellationToken::new();
|
||||
@@ -166,6 +285,21 @@ async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
tokio::time::timeout(Duration::from_secs(2), shutdown.cancelled())
|
||||
.await
|
||||
.expect("shutdown requested after spawn");
|
||||
#[cfg(windows)]
|
||||
{
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !fixture.root.path().join("replacement-waiting").exists() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replacement waiting for old service");
|
||||
assert!(
|
||||
!replacement_marker.exists(),
|
||||
"new service must wait for resource release"
|
||||
);
|
||||
}
|
||||
drop(service);
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !replacement_marker.is_file() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
@@ -189,7 +323,7 @@ async fn restart_should_use_startup_path_after_running_executable_is_renamed() {
|
||||
let output = std::process::Command::new(child)
|
||||
.args([
|
||||
"--exact",
|
||||
"system_update::restart_should_use_startup_path_after_running_executable_is_renamed",
|
||||
"system_update::operations::restart_should_use_startup_path_after_running_executable_is_renamed",
|
||||
])
|
||||
.env(CHILD_ENV, "1")
|
||||
.env("CPR_RESTART_DELAY_MS", "1200")
|
||||
@@ -223,7 +357,7 @@ async fn restart_should_use_startup_path_after_running_executable_is_renamed() {
|
||||
config.self_restart_enabled = true;
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
|
||||
|
||||
let backup = fixture.root.path().join("codex-proxy-rs.backup");
|
||||
let backup = fixture.backup();
|
||||
fs::rename(fixture.executable(), &backup).expect("back up running executable");
|
||||
fixture.write_executable("#!/bin/sh\nprintf new > \"${0%/*}/replacement-version\"\n");
|
||||
assert_eq!(std::env::current_exe().expect("renamed executable"), backup);
|
||||
@@ -777,24 +911,42 @@ async fn update_should_reject_release_archive_from_untrusted_host() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_reject_release_archive_with_unsafe_path() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::UnsafePath,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
for unsafe_path in [
|
||||
"../escape",
|
||||
#[cfg(windows)]
|
||||
"web/dist/index.html:payload",
|
||||
#[cfg(windows)]
|
||||
"web/dist/NUL.txt",
|
||||
#[cfg(windows)]
|
||||
"web/dist/COM¹",
|
||||
#[cfg(windows)]
|
||||
"web/dist/trailing. ",
|
||||
#[cfg(windows)]
|
||||
"C:/escape",
|
||||
] {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::UnsafePath(unsafe_path),
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
.await
|
||||
.operation
|
||||
.status,
|
||||
SystemOperationStatus::Failed,
|
||||
);
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
.await
|
||||
.operation
|
||||
.status,
|
||||
SystemOperationStatus::Failed,
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("unchanged binary"),
|
||||
b"old-binary"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -933,6 +1085,7 @@ async fn update_and_rollback_should_use_the_default_api_asset_directory() {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn update_should_replace_web_assets_across_filesystems() {
|
||||
let Ok(external) = tempfile::tempdir_in("/dev/shm") else {
|
||||
@@ -980,7 +1133,7 @@ async fn update_should_restore_web_assets_when_binary_backup_fails() {
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
fs::create_dir(fixture.executable().with_extension("backup")).expect("blocking backup dir");
|
||||
fs::create_dir(fixture.backup()).expect("blocking backup dir");
|
||||
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
|
||||
@@ -12,7 +12,7 @@ use gateway_admin::{
|
||||
instances::PluginInstance,
|
||||
state::{
|
||||
DeletePluginState, PluginStateConfiguration, PluginStateOwner,
|
||||
PluginStateOwnerRequest, PluginStateSchema, PutPluginState,
|
||||
PluginStateOwnerRequest, PutPluginState,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -25,7 +25,6 @@ use gateway_plugin_sdk::{
|
||||
StatePutRequest, StatePutResult, StateRecord,
|
||||
},
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::RpcReply;
|
||||
|
||||
@@ -43,7 +42,7 @@ impl PluginPrivateState {
|
||||
configuration: PluginStateConfiguration,
|
||||
owner: Option<PluginStateOwner>,
|
||||
) -> Result<Self, AdminError> {
|
||||
let validators = validators(manifest)?;
|
||||
let validators = crate::package::state_validators(manifest)?;
|
||||
Ok(Self {
|
||||
store,
|
||||
configuration,
|
||||
@@ -175,46 +174,6 @@ impl PluginPrivateState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn configuration(manifest: &Manifest) -> Result<PluginStateConfiguration, AdminError> {
|
||||
let _ = validators(manifest)?;
|
||||
let mut namespaces = Vec::with_capacity(manifest.state.len());
|
||||
for state in &manifest.state {
|
||||
let mut schema = state.schema.clone();
|
||||
schema.sort_all_objects();
|
||||
let encoded = serde_json::to_vec(&schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无法编码"))?;
|
||||
namespaces.push(PluginStateSchema {
|
||||
namespace: state.namespace.clone(),
|
||||
schema_version: state.schema_version,
|
||||
schema_sha256: hex::encode(Sha256::digest(encoded)),
|
||||
schema: state.schema.clone(),
|
||||
maximum_records: state.maximum_records,
|
||||
maximum_bytes: state.maximum_bytes,
|
||||
maximum_value_bytes: state.maximum_value_bytes,
|
||||
migrates_from: state.migrates_from.clone(),
|
||||
});
|
||||
}
|
||||
namespaces.sort_by(|left, right| left.namespace.cmp(&right.namespace));
|
||||
Ok(PluginStateConfiguration { namespaces })
|
||||
}
|
||||
|
||||
fn validators(manifest: &Manifest) -> Result<BTreeMap<String, jsonschema::Validator>, AdminError> {
|
||||
manifest
|
||||
.state
|
||||
.iter()
|
||||
.map(|state| {
|
||||
let validator = jsonschema::options()
|
||||
.offline()
|
||||
.with_pattern_options(
|
||||
jsonschema::PatternOptions::fancy_regex().backtrack_limit(20_000),
|
||||
)
|
||||
.build(&state.schema)
|
||||
.map_err(|_| AdminError::invalid("插件状态 schema 无效或引用了外部资源"))?;
|
||||
Ok((state.namespace.clone(), validator))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn map_store_fault(error: PluginStateStoreError) -> PluginFault {
|
||||
let (code, message) = match error.kind() {
|
||||
PluginStateStoreErrorKind::Invalid => (ErrorCode::InvalidInput, "state input is invalid"),
|
||||
|
||||
@@ -103,7 +103,7 @@ pub(super) fn validate(
|
||||
}
|
||||
Ok((
|
||||
configuration,
|
||||
crate::callback::private_state::configuration(manifest)?,
|
||||
crate::package::state_configuration(manifest)?,
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
//! 将已发布与准备中的实例事实投影为运行诊断,不参与候选提交
|
||||
|
||||
use super::super::restart_circuit::RestartIdentity;
|
||||
use super::restart_circuit::RestartIdentity;
|
||||
use super::{
|
||||
PluginRuntime,
|
||||
set::{PreparedSet, instance_fingerprint},
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user