mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 16:01:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68b297fc88 | ||
|
|
1b96c775b2 | ||
|
|
b9ccc0b702 | ||
|
|
f9e364860a | ||
|
|
ceeb68ba82 | ||
|
|
2abbfb1fac | ||
|
|
5b7f307bb2 | ||
|
|
e64fb2d955 | ||
|
|
7ee9ea8b63 | ||
|
|
ce12ba3f43 | ||
|
|
ac4ea71cd2 | ||
|
|
77de86e0b6 | ||
|
|
644296bf66 | ||
|
|
367a62bd7d | ||
|
|
62f321c82b | ||
|
|
c40e2f524e | ||
|
|
0f5bf03e13 | ||
|
|
28c8c536c5 | ||
|
|
4ee9abf9d2 | ||
|
|
779ccf4e6c | ||
|
|
d13a4a2138 | ||
|
|
a0e7f38f7f | ||
|
|
41f97ac836 | ||
|
|
975b73b33b | ||
|
|
d787195f4e | ||
|
|
8681d6851e | ||
|
|
d99ecf9aaf | ||
|
|
8f40bfda7c | ||
|
|
72215fe62a | ||
|
|
f61995e61d | ||
|
|
f85d80d756 | ||
|
|
e1f84c210b | ||
|
|
0e02a056b5 | ||
|
|
1eac929e9d | ||
|
|
9b4a042be5 | ||
|
|
3bd042181e | ||
|
|
2cae28d7f4 | ||
|
|
bab781cbe0 | ||
|
|
28ad8fb1c4 | ||
|
|
4888681a04 | ||
|
|
9739bb08be | ||
|
|
fac37ec563 | ||
|
|
9f0d712671 | ||
|
|
75be1287ef | ||
|
|
dad91fe9f7 | ||
|
|
b22b0cfa41 | ||
|
|
bd69329a37 | ||
|
|
151a3c6bfd | ||
|
|
e2f194875f | ||
|
|
07b61bdc83 | ||
|
|
c0fdc1bbff | ||
|
|
343c579017 | ||
|
|
914c00785c | ||
|
|
b3de0d637d | ||
|
|
1271fa0186 | ||
|
|
055da40476 | ||
|
|
36ba79ca5e | ||
|
|
1b24029a97 | ||
|
|
431992a7a4 | ||
|
|
d3143406dc | ||
|
|
478f2d1f21 |
@@ -125,7 +125,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
cache-bin: false
|
||||
save-if: ${{ github.event_name != 'pull_request' }}
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!/frontend/.env
|
||||
|
||||
# Editors and operating systems
|
||||
/.idea/
|
||||
|
||||
Generated
+106
-60
@@ -204,7 +204,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -266,9 +266,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.10.2"
|
||||
version = "1.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a"
|
||||
checksum = "de8930c11353c72ae54aa24dc94d692df74d59814839cbaf8069d63dbcb4c519"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -292,9 +292,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.157.0"
|
||||
version = "1.159.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8"
|
||||
checksum = "8d5c431c5db2f1f6f536c2a19425575e28273859c6ab399d9ac957f73678c7db"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -318,7 +318,7 @@ dependencies = [
|
||||
"hmac",
|
||||
"http",
|
||||
"http-body",
|
||||
"lru",
|
||||
"lru 0.18.5",
|
||||
"percent-encoding",
|
||||
"regex-lite",
|
||||
"sha2",
|
||||
@@ -328,9 +328,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.4"
|
||||
version = "1.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
checksum = "57a4393e6addc3380632887b0998d06ad5f099ab5c478be0a3eeedecf8604161"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -361,9 +361,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-checksums"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307"
|
||||
checksum = "ab2f2d12a431aab3703d55277278bad45f02208569b11506220b95fdc1ef0507"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-types",
|
||||
@@ -449,18 +449,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-observability"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c"
|
||||
checksum = "a416257d7b1b0f32a517a470a8c1dd354d8c4aa697ccab81a11a1e6cfb7418e3"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.2"
|
||||
version = "1.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
checksum = "5bc688673e282f42d3152dcd9cacc1a9a75734d7cd39d02e58989276fc750f3c"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -481,9 +483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.2"
|
||||
version = "1.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
checksum = "0467a2414e84b1b01e37452896fb3562b93926e81c619eba6ecf27ce9e682a0e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -509,9 +511,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "1.1.2"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9543fc722ed4865ab29fb66bd7ed0bef7b029d265ba8c587144511e30006d431"
|
||||
checksum = "f7ef30d39e131651359b60a1c560de9a9502a4c3754172ab2d6e8734fe21bcba"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -520,9 +522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.1"
|
||||
version = "1.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
checksum = "aff4bf014c512840b2bba80a20417b723cbd6a0a65c02e2d81f50b4268aedfc7"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -532,7 +534,6 @@ dependencies = [
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
"ryu",
|
||||
"serde",
|
||||
@@ -870,7 +871,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -924,7 +925,7 @@ dependencies = [
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -992,6 +993,15 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.2"
|
||||
@@ -1098,9 +1108,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.4"
|
||||
version = "1.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
|
||||
checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
@@ -1468,6 +1478,17 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a2e16b9a584b602f1e0e5e22c30ca0906786088c27500653e432d086def8b53"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.4.1"
|
||||
@@ -1511,9 +1532,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
version = "1.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide 0.9.1",
|
||||
@@ -1671,7 +1692,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1735,7 +1756,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1807,6 +1828,17 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gateway-egress"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"gateway-core",
|
||||
"http",
|
||||
"hyper-util",
|
||||
"reqwest",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gateway-host"
|
||||
version = "0.0.0"
|
||||
@@ -1822,6 +1854,7 @@ dependencies = [
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-egress",
|
||||
"gateway-protocol",
|
||||
"hex",
|
||||
"http",
|
||||
@@ -2514,15 +2547,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
checksum = "0e37f27df77c7887edbc5f5a59ecd8e5ec4a669319266ec1b61068667a603183"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
"data-encoding",
|
||||
"email_address",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.20.2",
|
||||
"fraction",
|
||||
"getrandom 0.3.4",
|
||||
"itoa",
|
||||
@@ -2542,18 +2575,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
checksum = "7e41151614b6c927e21982be8bef5a26741ae5a0ae6a781eedee2aa930a980e6"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
checksum = "1515b9a91134ef67d7bac95c2d76fbc7e383c4c55ee544204ceec8088f881fef"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2648,9 +2681,18 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.18.3"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c"
|
||||
checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
@@ -3296,7 +3338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3353,6 +3395,7 @@ dependencies = [
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-egress",
|
||||
"gateway-host",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.4.3",
|
||||
@@ -3361,7 +3404,7 @@ dependencies = [
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"lru",
|
||||
"lru 0.19.0",
|
||||
"native-tls",
|
||||
"openssl",
|
||||
"openssl-sys",
|
||||
@@ -3410,6 +3453,7 @@ dependencies = [
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"gateway-egress",
|
||||
"gateway-protocol",
|
||||
"getrandom 0.4.3",
|
||||
"jsonwebtoken",
|
||||
@@ -3478,9 +3522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.18"
|
||||
version = "0.11.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc"
|
||||
checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.4.3",
|
||||
@@ -3500,9 +3544,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quinn-udp"
|
||||
version = "0.5.15"
|
||||
version = "0.5.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
|
||||
checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016"
|
||||
dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -3659,14 +3703,14 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
checksum = "a028057ef852c58a42d75e09da87b5e733f4030f4c7411728955e4f87203a577"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3813,9 +3857,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
@@ -4029,7 +4073,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4511,9 +4555,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
version = "3.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -4611,7 +4655,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4632,7 +4676,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"bstr",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.19.2",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
]
|
||||
@@ -5096,9 +5140,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.29.0"
|
||||
version = "1.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5701,10 +5745,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.4.0"
|
||||
version = "9.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cec733c9820347a592f6375c3c0089b479ba0a0b1cbf3dd1958b02959b7c033"
|
||||
checksum = "e3476fd497ac8af5b1ec7b51acab7bef380f7e8dd8e0487595bafd839417a302"
|
||||
dependencies = [
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
@@ -5714,13 +5759,14 @@ dependencies = [
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.6"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
|
||||
@@ -5,6 +5,7 @@ members = [
|
||||
"crates/gateway-admin",
|
||||
"crates/gateway-api",
|
||||
"crates/gateway-core",
|
||||
"crates/gateway-egress",
|
||||
"crates/gateway-host",
|
||||
"crates/gateway-plugin/runtime",
|
||||
"crates/gateway-plugin/sdk",
|
||||
|
||||
@@ -16,6 +16,7 @@ pub(super) const WORKSPACE_MEMBERS: &[&str] = &[
|
||||
"crates/gateway-admin",
|
||||
"crates/gateway-api",
|
||||
"crates/gateway-core",
|
||||
"crates/gateway-egress",
|
||||
"crates/gateway-host",
|
||||
"crates/gateway-plugin/runtime",
|
||||
"crates/gateway-plugin/sdk",
|
||||
@@ -80,6 +81,7 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
"account/selection.rs" => Some(&["account", "concurrency", "identity", "validation"]),
|
||||
"concurrency.rs" => Some(&["error"]),
|
||||
"account/store.rs" => Some(&["account", "error", "identity", "validation"]),
|
||||
"account/proxy.rs" => Some(&["upstream"]),
|
||||
path if path.starts_with("policy/") => {
|
||||
Some(&["account", "identity", "policy", "validation"])
|
||||
}
|
||||
@@ -447,6 +449,7 @@ const PACKAGE_TO_MEMBER: &[(&str, &str)] = &[
|
||||
("gateway-admin", "crates/gateway-admin"),
|
||||
("gateway-api", "crates/gateway-api"),
|
||||
("gateway-core", "crates/gateway-core"),
|
||||
("gateway-egress", "crates/gateway-egress"),
|
||||
("gateway-host", "crates/gateway-host"),
|
||||
("gateway-plugin-runtime", "crates/gateway-plugin/runtime"),
|
||||
("gateway-plugin-sdk", "crates/gateway-plugin/sdk"),
|
||||
@@ -510,8 +513,10 @@ const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
("gateway-api", "gateway-core"),
|
||||
("gateway-api", "gateway-protocol"),
|
||||
("gateway-core", "gateway-protocol"),
|
||||
("gateway-egress", "gateway-core"),
|
||||
("gateway-host", "gateway-admin"),
|
||||
("gateway-host", "gateway-core"),
|
||||
("gateway-host", "gateway-egress"),
|
||||
("gateway-host", "gateway-protocol"),
|
||||
("gateway-plugin-runtime", "gateway-host"),
|
||||
("gateway-plugin-runtime", "gateway-core"),
|
||||
@@ -522,9 +527,11 @@ const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
("gateway-store", "gateway-core"),
|
||||
("provider-openai", "gateway-admin"),
|
||||
("provider-openai", "gateway-core"),
|
||||
("provider-openai", "gateway-egress"),
|
||||
("provider-openai", "gateway-protocol"),
|
||||
("provider-xai", "gateway-admin"),
|
||||
("provider-xai", "gateway-core"),
|
||||
("provider-xai", "gateway-egress"),
|
||||
("provider-xai", "gateway-protocol"),
|
||||
];
|
||||
|
||||
|
||||
@@ -78,7 +78,12 @@ async fn proxy_probe_should_use_provider_custom_ca_for_https_proxies() {
|
||||
});
|
||||
let probe = HttpProxyProbe::new("http://unresolvable.invalid/ip")
|
||||
.with_client_builder(provider_openai::build_reqwest_client_with_custom_ca);
|
||||
let result = probe.test(Some(&proxy), false).await;
|
||||
let result = probe
|
||||
.test(
|
||||
gateway_admin::ports::proxy::ProbeRoute::Proxy(&proxy),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(result.success, "{}", result.message);
|
||||
assert_eq!(result.exit_ip.unwrap().to_string(), "203.0.113.8");
|
||||
tokio::time::timeout(Duration::from_secs(5), server)
|
||||
|
||||
@@ -21,7 +21,7 @@ pub use gateway_core::account::{
|
||||
/// 账号可用的管理操作
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderAccountCapabilities {
|
||||
pub quota_limits_supported: bool,
|
||||
pub quota_settings_supported: bool,
|
||||
pub quota_credits: bool,
|
||||
pub quota: bool,
|
||||
pub quota_refresh: bool,
|
||||
@@ -113,7 +113,7 @@ pub enum AccountGroupFilter {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountRecord {
|
||||
pub quota_paused: bool,
|
||||
pub quota_limits: gateway_core::account::AccountQuotaLimits,
|
||||
pub quota_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub auto_use_quota_credits: bool,
|
||||
pub id: String,
|
||||
pub provider_kind: ProviderKind,
|
||||
@@ -146,7 +146,7 @@ pub struct AccountRecord {
|
||||
}
|
||||
|
||||
impl AccountRecord {
|
||||
/// 配置保留绑定,执行请求时禁用代理按直连处理
|
||||
/// 配置保留绑定,执行请求时禁用代理使用部署默认出口
|
||||
#[must_use]
|
||||
pub fn effective_outbound_proxy(&self) -> Option<&gateway_core::account::OutboundProxy> {
|
||||
self.outbound_proxy
|
||||
@@ -254,7 +254,7 @@ pub struct AccountUpdateResult {
|
||||
/// 仅修改显式字段的批量账号设置命令
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BatchUpdateAccounts {
|
||||
pub quota_limits: Option<gateway_core::account::AccountQuotaLimits>,
|
||||
pub quota_settings: Option<gateway_core::account::AccountQuotaSettings>,
|
||||
pub auto_use_quota_credits: Option<bool>,
|
||||
pub account_ids: Vec<String>,
|
||||
pub enabled: Option<bool>,
|
||||
@@ -268,6 +268,8 @@ pub struct BatchUpdateAccounts {
|
||||
/// 已按拟提交策略重算的账号暂停结果,以快照版本防止旧判断覆盖新观测
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PreparedQuotaPause {
|
||||
pub settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub previous_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub revision: Revision,
|
||||
pub paused: bool,
|
||||
|
||||
@@ -24,7 +24,7 @@ impl RetentionPolicy {
|
||||
audit_days: u32,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
((1..=31).contains(&usage_days), "usage_retention_days"),
|
||||
(usage_days > 0, "usage_retention_days"),
|
||||
(ops_days > 0, "ops_event_retention_days"),
|
||||
(audit_days > 0, "audit_retention_days"),
|
||||
] {
|
||||
@@ -38,7 +38,7 @@ impl RetentionPolicy {
|
||||
pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result<Self, AdminError> {
|
||||
if Self::validate_values(usage_days, ops_days, audit_days).is_err() {
|
||||
return Err(AdminError::invalid(
|
||||
"请求保留期必须为 1~31 天,事件与审计保留期必须为正数",
|
||||
"请求、事件与审计保留期必须为正整数天数",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
|
||||
@@ -63,6 +63,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: gateway_core::account::ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
|
||||
@@ -69,8 +69,14 @@ pub struct ProxyImportReservation {
|
||||
pub guard: Box<dyn ProxyImportGuard>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum ProbeRoute<'a> {
|
||||
Direct,
|
||||
Deployment,
|
||||
Proxy(&'a OutboundProxy),
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProxyProbe: Send + Sync {
|
||||
/// 未指定代理时探测服务器直连出口,不继承进程代理环境变量
|
||||
async fn test(&self, proxy: Option<&OutboundProxy>, detect_location: bool) -> ProxyTestResult;
|
||||
async fn test(&self, proxy: ProbeRoute<'_>, detect_location: bool) -> ProxyTestResult;
|
||||
}
|
||||
|
||||
@@ -759,7 +759,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
self.accounts
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec![account_id.to_string()],
|
||||
enabled: Some(true),
|
||||
@@ -846,20 +846,19 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut pauses = BTreeMap::new();
|
||||
for account_id in &account_ids {
|
||||
let (mut item, provider) = self.provider_for_account(account_id).await?;
|
||||
if let Some(limits) = &command.quota_limits {
|
||||
if let Some(settings) = &command.quota_settings {
|
||||
if !provider
|
||||
.account_capabilities(account_id, &item.account.authentication_kind)
|
||||
.quota_limits_supported
|
||||
.quota_settings_supported
|
||||
{
|
||||
return Err(AdminError::invalid("该账号不支持通用额度窗口上限"));
|
||||
return Err(AdminError::invalid("该账号不支持额度桶设置"));
|
||||
}
|
||||
let new_keys = limits
|
||||
.keys()
|
||||
.filter(|key| !item.account.quota_limits.contains(key))
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
if !limits.is_empty() {
|
||||
item.account.quota_limits = limits.clone();
|
||||
let previous_settings = item.account.quota_settings.clone();
|
||||
let settings = settings.clone();
|
||||
let mut observed_at = None;
|
||||
let mut paused = false;
|
||||
if !settings.is_empty() {
|
||||
item.account.quota_settings = settings.clone();
|
||||
let quota = provider
|
||||
.read_quotas(&[ProviderQuotaReadRequest {
|
||||
account: &item.account,
|
||||
@@ -870,24 +869,31 @@ impl AccountsService for DefaultAccountsService {
|
||||
.next()
|
||||
.ok_or_else(|| AdminError::invalid("缺少账号额度观测结果"))?
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
if new_keys.iter().any(|key| {
|
||||
!quota.windows.iter().any(|window| {
|
||||
window.key == *key
|
||||
&& window.local_usage_attribution
|
||||
== QuotaLocalUsageAttribution::AccountWide
|
||||
})
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能设置已观测的通用额度窗口"));
|
||||
for key in settings.keys() {
|
||||
let window = quota.windows.iter().find(|window| window.key == key);
|
||||
if window.is_none() && !previous_settings.contains(key) {
|
||||
return Err(AdminError::invalid("只能设置已观测的额度桶"));
|
||||
}
|
||||
if window.is_some_and(|window| {
|
||||
window.local_usage_attribution
|
||||
!= QuotaLocalUsageAttribution::AccountWide
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能设置通用额度桶"));
|
||||
}
|
||||
}
|
||||
pauses.insert(
|
||||
item.account.id.clone(),
|
||||
crate::model::accounts::PreparedQuotaPause {
|
||||
revision: item.account.credential_revision,
|
||||
observed_at: quota.observed_at,
|
||||
paused: !quota.paused_windows.is_empty(),
|
||||
},
|
||||
);
|
||||
observed_at = quota.observed_at;
|
||||
paused = !quota.paused_windows.is_empty();
|
||||
}
|
||||
pauses.insert(
|
||||
item.account.id.clone(),
|
||||
crate::model::accounts::PreparedQuotaPause {
|
||||
settings,
|
||||
previous_settings,
|
||||
revision: item.account.credential_revision,
|
||||
observed_at,
|
||||
paused,
|
||||
},
|
||||
);
|
||||
}
|
||||
if command.auto_use_quota_credits.is_some()
|
||||
&& !provider
|
||||
|
||||
@@ -52,7 +52,7 @@ impl PeerInfoService {
|
||||
});
|
||||
let limits = peer.and_then(|peer| peer.key_limits);
|
||||
let info = ProviderUpstreamInfo {
|
||||
discovered: peer.is_some() || stored_failure == Some(UpstreamInfoFailure::Unauthorized),
|
||||
discovered: snapshot::is_discovered(peer, stored_failure),
|
||||
observed_at,
|
||||
stale,
|
||||
connection_revision: connection.revision.clone(),
|
||||
|
||||
@@ -282,6 +282,21 @@ impl PeerInfoService {
|
||||
}
|
||||
|
||||
impl UpstreamInfoSource for PeerInfoService {
|
||||
fn is_peer(&self, account: &ProviderAccount) -> bool {
|
||||
if !self.supported(account) {
|
||||
return false;
|
||||
}
|
||||
let connection = UpstreamConnection::for_account(account);
|
||||
self.observations
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(account.id())
|
||||
.is_some_and(|cached| {
|
||||
cached.connection == connection
|
||||
&& snapshot::is_discovered(cached.summary.as_deref(), cached.stored_failure)
|
||||
})
|
||||
}
|
||||
|
||||
fn signal(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
|
||||
@@ -16,6 +16,10 @@ use serde::Deserialize as _;
|
||||
pub(super) const MAX_AGE: Duration = Duration::from_secs(30 * 60);
|
||||
pub(super) const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
pub(super) fn is_discovered(peer: Option<&PeerLink>, failure: Option<UpstreamInfoFailure>) -> bool {
|
||||
peer.is_some() || failure == Some(UpstreamInfoFailure::Unauthorized)
|
||||
}
|
||||
|
||||
// 不支持协议是连接的识别结果,复用观测存储与版本保护,不伪造上游额度快照
|
||||
pub(super) fn unsupported(connection: &UpstreamConnection) -> OpaqueProviderData {
|
||||
OpaqueProviderData::new(serde_json::Map::from_iter([
|
||||
|
||||
@@ -306,7 +306,9 @@ impl PluginsService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("下载代理已禁用,请启用或改为直连"));
|
||||
return Err(AdminError::invalid(
|
||||
"下载代理已禁用,请启用或改用部署默认出口",
|
||||
));
|
||||
}
|
||||
Ok(Some(PluginDistributionEgress::new(
|
||||
record.id,
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::{
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
proxy::{PluginProxyAccess, ProxyProbe, ProxyStore},
|
||||
proxy::{PluginProxyAccess, ProbeRoute, ProxyProbe, ProxyStore},
|
||||
store::AdminStoreErrorKind,
|
||||
},
|
||||
};
|
||||
@@ -47,7 +47,7 @@ impl PluginProxyAccess for DefaultPluginProxyAccess {
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
// 显式选择代理时必须有可用配置,不能沿用账号停用代理后直连的语义
|
||||
// 显式选择代理时必须有可用配置,不能沿用账号停用代理后使用部署默认出口的语义
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("代理已禁用,请启用后再选择"));
|
||||
}
|
||||
@@ -114,10 +114,10 @@ pub trait ProxiesService: Send + Sync {
|
||||
detect_location: bool,
|
||||
context: &MutationContext,
|
||||
) -> Result<ProxyRecord, AdminError>;
|
||||
/// 探测未保存的连接地址或服务器直连出口,不写入配置或修改账号绑定
|
||||
/// 探测未保存的连接地址、部署默认出口或服务器直连出口,不写入配置或修改账号绑定
|
||||
async fn probe(
|
||||
&self,
|
||||
proxy: Option<&OutboundProxy>,
|
||||
proxy: ProbeRoute<'_>,
|
||||
detect_location: bool,
|
||||
) -> Result<ProxyTestResult, AdminError>;
|
||||
}
|
||||
@@ -323,7 +323,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
.transpose()
|
||||
.map_err(|_| AdminError::invalid("代理位置不合法"))?;
|
||||
command.test = if command.auto_location {
|
||||
Some(self.probe(Some(&command.proxy), true).await?)
|
||||
Some(self.probe(ProbeRoute::Proxy(&command.proxy), true).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -362,7 +362,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
if command.auto_location.unwrap_or(current.auto_location)
|
||||
&& (!current.auto_location || proxy != ¤t.proxy)
|
||||
{
|
||||
command.test = Some(self.probe(Some(proxy), true).await?);
|
||||
command.test = Some(self.probe(ProbeRoute::Proxy(proxy), true).await?);
|
||||
}
|
||||
}
|
||||
let result = self
|
||||
@@ -391,7 +391,7 @@ impl ProxiesService for DefaultProxiesService {
|
||||
|
||||
async fn probe(
|
||||
&self,
|
||||
proxy: Option<&OutboundProxy>,
|
||||
proxy: ProbeRoute<'_>,
|
||||
detect_location: bool,
|
||||
) -> Result<ProxyTestResult, AdminError> {
|
||||
let _permit = self
|
||||
@@ -423,7 +423,10 @@ impl ProxiesService for DefaultProxiesService {
|
||||
// 手动解析只为本次测试请求位置,不改变自动跟随的持久配置
|
||||
let result = self
|
||||
.probe
|
||||
.test(Some(&record.proxy), record.auto_location || detect_location)
|
||||
.test(
|
||||
ProbeRoute::Proxy(&record.proxy),
|
||||
record.auto_location || detect_location,
|
||||
)
|
||||
.await;
|
||||
let mutation = self
|
||||
.store
|
||||
|
||||
@@ -3,25 +3,15 @@
|
||||
use gateway_admin::model::retention::{RetentionPolicy, RetentionTarget};
|
||||
|
||||
#[test]
|
||||
fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows() {
|
||||
for (usage, ops, audit) in [
|
||||
(0, 1, 1),
|
||||
(32, 1, 1),
|
||||
(u32::MAX, 1, 1),
|
||||
(1, 0, 1),
|
||||
(1, 1, 0),
|
||||
] {
|
||||
fn retention_policy_rejects_zero_for_each_target() {
|
||||
for (usage, ops, audit) in [(0, 1, 1), (1, 0, 1), (1, 1, 0)] {
|
||||
assert!(RetentionPolicy::try_new(usage, ops, audit).is_err());
|
||||
}
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
assert_eq!(policy.days(RetentionTarget::ModelRequests), 31);
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 7);
|
||||
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_retention_accepts_short_windows_without_changing_other_targets() {
|
||||
for days in [1, 3, 7, 15, 30, 31] {
|
||||
fn request_retention_accepts_positive_windows_without_changing_other_targets() {
|
||||
for days in [1, 31, 32, 365, 3650, u32::MAX] {
|
||||
let policy = RetentionPolicy::try_new(days, 30, 90).unwrap();
|
||||
assert_eq!(policy.days(RetentionTarget::ModelRequests), days);
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 30);
|
||||
|
||||
@@ -1180,6 +1180,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 30,
|
||||
@@ -1913,7 +1914,7 @@ async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
|
||||
.batch_update(
|
||||
&context("batch-update-request"),
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
@@ -3412,7 +3413,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -140,6 +140,10 @@ impl PendingPeer {
|
||||
}
|
||||
|
||||
impl UpstreamInfoSource for PendingPeer {
|
||||
fn is_peer(&self, _: &ProviderAccount) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn signal(
|
||||
&self,
|
||||
_: &ProviderAccount,
|
||||
|
||||
@@ -51,6 +51,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -1432,6 +1432,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -239,6 +239,7 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
initialize_peer_upstream(accounts.clone(), ®istry, transport.clone()).unwrap();
|
||||
let initial_accounts = accounts.accounts.lock().unwrap().clone();
|
||||
let reader = UpstreamInfoReader::default();
|
||||
assert!(!reader.is_peer(&initial_accounts[0]));
|
||||
assert!(
|
||||
reader
|
||||
.signal(&initial_accounts[0], Some("model"), SystemTime::now())
|
||||
@@ -246,6 +247,7 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
);
|
||||
let source = bundle.scheduling_source();
|
||||
reader.bind(&source).unwrap();
|
||||
assert!(!reader.is_peer(&initial_accounts[0]));
|
||||
assert!(reader.bind(&source).is_err());
|
||||
drop(source);
|
||||
let WorkerContribution::Registration(worker) = bundle.take_worker_contributions().remove(0)
|
||||
@@ -272,6 +274,10 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
}
|
||||
);
|
||||
for account in &initial_accounts[..2] {
|
||||
assert_eq!(
|
||||
reader.is_peer(account),
|
||||
Some(account.provider().as_str()) != failed_provider
|
||||
);
|
||||
if Some(account.provider().as_str()) == failed_provider {
|
||||
assert!(
|
||||
reader
|
||||
@@ -304,6 +310,7 @@ async fn discovery_uses_provider_capabilities_and_publishes_neutral_signals() {
|
||||
);
|
||||
drop(worker);
|
||||
drop(bundle);
|
||||
assert!(!reader.is_peer(&initial_accounts[0]));
|
||||
assert!(
|
||||
reader
|
||||
.signal(&initial_accounts[0], Some("model"), SystemTime::now())
|
||||
@@ -340,6 +347,7 @@ async fn account_edits_preserve_observation_and_refresh_budget_until_connection_
|
||||
.is_some()
|
||||
);
|
||||
let after = service.refresh_quota(edited.id()).await.unwrap();
|
||||
assert!(bundle.scheduling_source().is_peer(&edited));
|
||||
assert_eq!(after.upstream, before.upstream);
|
||||
assert_eq!(transport.0.lock().unwrap().len(), 1);
|
||||
|
||||
@@ -347,6 +355,7 @@ async fn account_edits_preserve_observation_and_refresh_budget_until_connection_
|
||||
OutboundProxy::parse("http://proxy.invalid:8080").unwrap(),
|
||||
));
|
||||
*accounts.accounts.lock().unwrap() = vec![reconnected.clone()];
|
||||
assert!(!bundle.scheduling_source().is_peer(&reconnected));
|
||||
service.account_changed(reconnected.id()).await.unwrap();
|
||||
assert!(
|
||||
bundle
|
||||
@@ -422,6 +431,21 @@ fn loaded(account: &ProviderAccount) -> LoadedCredential {
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for TestAccounts {
|
||||
async fn load_account_catalogs(
|
||||
&self,
|
||||
_: &gateway_core::routing::ProviderKind,
|
||||
) -> Result<Vec<gateway_core::account::StoredAccountCatalog>, gateway_core::error::StoreError>
|
||||
{
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn save_account_catalog(
|
||||
&self,
|
||||
_: &gateway_core::account::AccountCatalogObservation,
|
||||
_: &[u8],
|
||||
) -> Result<bool, gateway_core::error::StoreError> {
|
||||
unreachable!("fixture does not discover account catalogs")
|
||||
}
|
||||
|
||||
async fn create_account(&self, _account: NewProviderAccount) -> Result<(), StoreError> {
|
||||
panic!("unused account operation")
|
||||
}
|
||||
|
||||
@@ -215,6 +215,10 @@ async fn known_peer_identity_changes_only_on_definitive_detection() {
|
||||
quota.upstream
|
||||
);
|
||||
let account = accounts.accounts.lock().unwrap()[0].clone();
|
||||
assert_eq!(
|
||||
bundle.scheduling_source().is_peer(&account),
|
||||
!clears_identity
|
||||
);
|
||||
assert_eq!(
|
||||
bundle
|
||||
.scheduling_source()
|
||||
@@ -237,6 +241,7 @@ async fn known_peer_identity_changes_only_on_definitive_detection() {
|
||||
drop(bundle);
|
||||
let restarted = initialize(accounts, transport);
|
||||
let restored = restarted.service().read_quota(&id).await.unwrap().unwrap();
|
||||
assert!(!restarted.scheduling_source().is_peer(&account));
|
||||
assert_eq!(
|
||||
restored.upstream.as_ref().unwrap().discovered,
|
||||
!clears_identity
|
||||
@@ -258,6 +263,10 @@ async fn known_peer_identity_changes_only_on_definitive_detection() {
|
||||
.upstream,
|
||||
restored.upstream
|
||||
);
|
||||
assert_eq!(
|
||||
restarted.scheduling_source().is_peer(&account),
|
||||
!clears_identity
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ use async_trait::async_trait;
|
||||
use gateway_admin::{
|
||||
model::{MutationContext, Revision, proxies::*},
|
||||
ports::{
|
||||
proxy::{ProxyProbe, ProxyStore},
|
||||
proxy::{ProbeRoute, ProxyProbe, ProxyStore},
|
||||
store::AdminStoreResult,
|
||||
},
|
||||
};
|
||||
@@ -194,7 +194,7 @@ async fn plugin_proxy_resolution_requires_enabled_configuration_without_account_
|
||||
|
||||
#[async_trait]
|
||||
impl ProxyProbe for TestProxies {
|
||||
async fn test(&self, _: Option<&OutboundProxy>, _: bool) -> ProxyTestResult {
|
||||
async fn test(&self, _: ProbeRoute<'_>, _: bool) -> ProxyTestResult {
|
||||
panic!("unexpected proxy probe")
|
||||
}
|
||||
}
|
||||
@@ -456,7 +456,7 @@ impl Drop for ActiveProbe<'_> {
|
||||
|
||||
#[async_trait]
|
||||
impl ProxyProbe for GatedProxyProbe {
|
||||
async fn test(&self, _: Option<&OutboundProxy>, _: bool) -> ProxyTestResult {
|
||||
async fn test(&self, _: ProbeRoute<'_>, _: bool) -> ProxyTestResult {
|
||||
self.active
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let _active = ActiveProbe(&self.active);
|
||||
@@ -526,8 +526,8 @@ async fn batch_tests_share_capacity_preserve_order_and_release_slots_on_cancella
|
||||
_ = async { receiver.recv().await.unwrap(); receiver.recv().await.unwrap(); } => {}
|
||||
}
|
||||
assert_eq!(probe.active.load(Ordering::Relaxed), 2);
|
||||
let mut single_a = Box::pin(services.proxies().probe(None, false));
|
||||
let mut single_b = Box::pin(services.proxies().probe(None, false));
|
||||
let mut single_a = Box::pin(services.proxies().probe(ProbeRoute::Direct, false));
|
||||
let mut single_b = Box::pin(services.proxies().probe(ProbeRoute::Direct, false));
|
||||
tokio::select! {
|
||||
_ = &mut batch => panic!("batch unexpectedly finished"),
|
||||
_ = &mut single_a => panic!("single probe unexpectedly finished"),
|
||||
@@ -538,7 +538,7 @@ async fn batch_tests_share_capacity_preserve_order_and_release_slots_on_cancella
|
||||
assert_eq!(
|
||||
services
|
||||
.proxies()
|
||||
.probe(None, false)
|
||||
.probe(ProbeRoute::Direct, false)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
|
||||
@@ -100,7 +100,9 @@ fn contract_type(mut item: Item, name: &str) -> Item {
|
||||
attrs.retain(|attr| attr.path().is_ident("serde"));
|
||||
attrs.insert(
|
||||
0,
|
||||
if name == "SmartSchedulingConfig" {
|
||||
if name == "ResetWindowWeights" {
|
||||
parse_quote!(#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)])
|
||||
} else if name == "SmartSchedulingConfig" {
|
||||
parse_quote!(#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)])
|
||||
} else if comparable {
|
||||
parse_quote!(#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)])
|
||||
@@ -255,8 +257,8 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
&["RequestLocation"][..],
|
||||
),
|
||||
(
|
||||
"../gateway-core/src/account/smart_scheduling.rs",
|
||||
&["SmartSchedulingValues"][..],
|
||||
"../gateway-core/src/account/scheduling.rs",
|
||||
&["SmartSchedulingValues", "ResetWindowValues"][..],
|
||||
),
|
||||
(
|
||||
"../gateway-core/src/metering/pricing.rs",
|
||||
@@ -324,10 +326,10 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
}
|
||||
types.push(contract_type(
|
||||
item,
|
||||
if *name == "SmartSchedulingValues" {
|
||||
"SmartSchedulingConfig"
|
||||
} else {
|
||||
name
|
||||
match *name {
|
||||
"SmartSchedulingValues" => "SmartSchedulingConfig",
|
||||
"ResetWindowValues" => "ResetWindowWeights",
|
||||
_ => name,
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
@@ -343,6 +343,8 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(
|
||||
),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -117,7 +117,7 @@ pub(super) fn account_view(
|
||||
let (mut quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now, time);
|
||||
quota.protection_rules = protection_rules;
|
||||
AccountView {
|
||||
quota_limits: account.quota_limits,
|
||||
quota_settings: account.quota_settings,
|
||||
auto_use_quota_credits: account.auto_use_quota_credits,
|
||||
upstream,
|
||||
capabilities: capabilities.into(),
|
||||
|
||||
@@ -61,7 +61,7 @@ pub struct ListQuery {
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct BatchUpdateAccountsRequest {
|
||||
pub quota_limits: Option<gateway_core::account::AccountQuotaLimits>,
|
||||
pub quota_settings: Option<gateway_core::account::AccountQuotaSettings>,
|
||||
pub auto_use_quota_credits: Option<bool>,
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
pub outbound_proxy_url: Option<AccountProxyUpdate>,
|
||||
@@ -119,7 +119,7 @@ impl BatchUpdateAccountsRequest {
|
||||
.map(parse_concurrency_limit)
|
||||
.transpose()?;
|
||||
self.weight.map(parse_account_weight).transpose()?;
|
||||
if self.quota_limits.is_none()
|
||||
if self.quota_settings.is_none()
|
||||
&& self.auto_use_quota_credits.is_none()
|
||||
&& self.enabled.is_none()
|
||||
&& self.concurrency_limit.is_none()
|
||||
@@ -137,7 +137,7 @@ impl BatchUpdateAccountsRequest {
|
||||
pub(super) fn into_command(self) -> Result<BatchUpdateAccounts, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(BatchUpdateAccounts {
|
||||
quota_limits: self.quota_limits,
|
||||
quota_settings: self.quota_settings,
|
||||
auto_use_quota_credits: self.auto_use_quota_credits,
|
||||
outbound_proxy: proxy_selection(self.outbound_proxy_id, self.outbound_proxy_url)?,
|
||||
account_ids: self.account_ids,
|
||||
@@ -273,7 +273,7 @@ pub struct AccountSummaryView {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountView {
|
||||
pub quota_limits: gateway_core::account::AccountQuotaLimits,
|
||||
pub quota_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub auto_use_quota_credits: bool,
|
||||
pub upstream: Option<super::AccountUpstreamInfoView>,
|
||||
pub capabilities: AccountCapabilitiesView,
|
||||
@@ -327,7 +327,7 @@ pub struct AccountCapacityView {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountCapabilitiesView {
|
||||
pub quota_limits_supported: bool,
|
||||
pub quota_settings_supported: bool,
|
||||
pub quota_credits: bool,
|
||||
pub quota: bool,
|
||||
pub quota_refresh: bool,
|
||||
@@ -347,7 +347,7 @@ impl From<gateway_admin::model::accounts::ProviderAccountCapabilities> for Accou
|
||||
subscription: value.subscription,
|
||||
avatar: value.avatar,
|
||||
reset_credits: value.reset_credits,
|
||||
quota_limits_supported: value.quota_limits_supported,
|
||||
quota_settings_supported: value.quota_settings_supported,
|
||||
quota_credits: value.quota_credits,
|
||||
consume_reset_credit: value.consume_reset_credit,
|
||||
}
|
||||
|
||||
@@ -101,6 +101,8 @@ struct TestRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct ProbeRequest {
|
||||
#[serde(default)]
|
||||
use_deployment_proxy: bool,
|
||||
#[serde(default)]
|
||||
detect_location: bool,
|
||||
proxy_url: AccountProxyUpdate,
|
||||
@@ -537,10 +539,21 @@ async fn probe<S>(
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
use gateway_admin::ports::proxy::ProbeRoute;
|
||||
let route = match (request.proxy_url.0.as_ref(), request.use_deployment_proxy) {
|
||||
(Some(_), true) => {
|
||||
return Err(AdminError::bad_request(
|
||||
"指定代理地址时不能同时选择部署默认出口",
|
||||
));
|
||||
}
|
||||
(Some(proxy), false) => ProbeRoute::Proxy(proxy),
|
||||
(None, true) => ProbeRoute::Deployment,
|
||||
(None, false) => ProbeRoute::Direct,
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.proxies()
|
||||
.probe(request.proxy_url.0.as_ref(), request.detect_location)
|
||||
.probe(route, request.detect_location)
|
||||
.await
|
||||
.map_err(map_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: gateway_core::account::ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
@@ -77,6 +78,7 @@ pub struct RuntimeSettingsView {
|
||||
pub values: RuntimeSettingsFields,
|
||||
pub config_revision: u64,
|
||||
pub smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights_defaults: gateway_core::account::ResetWindowWeights,
|
||||
pub provider_request_profiles: ProviderRequestProfiles,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub openai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
@@ -150,6 +152,8 @@ impl From<(RuntimeSettings, crate::time::TimePresenter)> for RuntimeSettingsView
|
||||
provider_request_profiles,
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights_defaults: gateway_core::account::ResetWindowWeights::default(
|
||||
),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
updated_at_display: time.datetime(&settings.updated_at),
|
||||
updated_at: settings.updated_at,
|
||||
@@ -658,7 +662,7 @@ where
|
||||
let result = state
|
||||
.admin_services()
|
||||
.proxies()
|
||||
.probe(None, true)
|
||||
.probe(gateway_admin::ports::proxy::ProbeRoute::Direct, true)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
@@ -765,6 +769,7 @@ impl RuntimeSettingsFields {
|
||||
openai_session_affinity_ttl_hours: self.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: self.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: self.smart_scheduling,
|
||||
quota_reset_window_weights: self.quota_reset_window_weights,
|
||||
min_codex_desktop_version: self.min_codex_desktop_version,
|
||||
min_codex_cli_version: self.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
@@ -808,6 +813,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
openai_session_affinity_ttl_hours: settings.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
quota_reset_window_weights: settings.quota_reset_window_weights,
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
|
||||
@@ -72,6 +72,10 @@ pub(crate) fn client_error_code(code: &str) -> &str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_retry_after_header(name: &str) -> bool {
|
||||
name.eq_ignore_ascii_case("retry-after") || name.eq_ignore_ascii_case("retry-after-ms")
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
@@ -87,6 +91,12 @@ pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
// 容量等待与换号已由网关执行;旧账号的等待提示不能再次阻塞客户端的新一轮路由
|
||||
for path in ["/headers", "/error/headers", "/response/error/headers"] {
|
||||
if let Some(Value::Object(headers)) = projected.pointer_mut(path) {
|
||||
headers.retain(|name, _| !is_retry_after_header(name));
|
||||
}
|
||||
}
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
|
||||
@@ -22,6 +22,7 @@ use gateway_core::{
|
||||
error::GatewayError,
|
||||
operation::Operation,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
|
||||
use crate::client_error::{ProtocolError, ProtocolErrorBody};
|
||||
use crate::middleware::headers::encode_headers;
|
||||
@@ -65,7 +66,7 @@ where
|
||||
Box::new(move |prepared, request| {
|
||||
Box::pin(async move {
|
||||
let (protocol, headers, body) = request_parts(request.clone())?;
|
||||
if protocol != "openai" {
|
||||
if protocol != OPENAI_PROTOCOL {
|
||||
return Err(MiddlewareError::Rejected);
|
||||
}
|
||||
let operation = request.apply_capabilities(decode(body, headers).await?)?;
|
||||
@@ -80,7 +81,8 @@ where
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
return buffered_response("openai", gateway_error_response(&error)).await;
|
||||
return buffered_response(OPENAI_PROTOCOL, gateway_error_response(&error))
|
||||
.await;
|
||||
}
|
||||
};
|
||||
collect_raw_json_response(started).await
|
||||
@@ -89,7 +91,9 @@ where
|
||||
)
|
||||
.await;
|
||||
let response = match result {
|
||||
Ok(response) => into_http_response(response, ExpectedBody::SingleJson, "openai").await,
|
||||
Ok(response) => {
|
||||
into_http_response(response, ExpectedBody::SingleJson, OPENAI_PROTOCOL).await
|
||||
}
|
||||
Err(error) => error_response(error),
|
||||
};
|
||||
if !execution_entered.load(Ordering::Relaxed) {
|
||||
@@ -143,7 +147,7 @@ async fn collect_raw_json_response(
|
||||
.await
|
||||
.map_err(|_| MiddlewareError::Fault)?;
|
||||
Ok(pending_execution_response(
|
||||
"openai".to_owned(),
|
||||
OPENAI_PROTOCOL.to_owned(),
|
||||
parts.status.as_u16(),
|
||||
encode_headers(&parts.headers),
|
||||
MiddlewareFrame::new(bytes, framing, true).with_transformed(transformed),
|
||||
@@ -155,7 +159,7 @@ fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == OPENAI_PROTOCOL) else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
|
||||
@@ -15,7 +15,7 @@ use serde_json::{Value, json};
|
||||
|
||||
pub use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client};
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client, is_retry_after_header};
|
||||
|
||||
pub use crate::client_error::{
|
||||
gateway_error_contract, gateway_error_response, openai_error_response,
|
||||
@@ -102,7 +102,9 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
/// 在下游尚未提交时交付 Provider 的 HTTP 失败响应,并应用客户端容量恢复合同
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let capacity_unavailable = matches!(error, EngineError::Provider(error)
|
||||
if error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
if error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable
|
||||
&& error.client_visible_upstream_error().and_then(|detail| detail.code())
|
||||
!= Some("previous_response_not_found"));
|
||||
if let EngineError::Provider(error) = error
|
||||
&& let Some(upstream) = error.client_visible_upstream_response()
|
||||
&& let Ok(status) = StatusCode::from_u16(upstream.status())
|
||||
@@ -110,6 +112,7 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let projected = serde_json::from_slice::<Value>(upstream.body())
|
||||
.ok()
|
||||
.and_then(|body| capacity_error_for_client(&body));
|
||||
let suppress_retry_after = capacity_unavailable || projected.is_some();
|
||||
let mut response = match projected {
|
||||
Some(body) => {
|
||||
let mut response = Response::new(Body::from(body.to_string()));
|
||||
@@ -132,6 +135,9 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
response.headers_mut().insert(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for header in upstream.headers() {
|
||||
if suppress_retry_after && is_retry_after_header(header.name()) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
@@ -140,6 +146,13 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
if !response.headers().contains_key("x-request-id")
|
||||
&& !response.headers().contains_key("x-oai-request-id")
|
||||
&& let Some(request_id) = error.upstream_request_id()
|
||||
&& let Ok(value) = HeaderValue::from_str(request_id.as_str())
|
||||
{
|
||||
response.headers_mut().insert("x-request-id", value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
let mut response = gateway_error_response(&gateway_error_from_engine(error));
|
||||
|
||||
@@ -32,6 +32,7 @@ use gateway_core::engine::{
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::lifecycle::{CancellationToken, ConnectionGuard};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use gateway_protocol::openai::sse::response_failed_sse_event_with_id;
|
||||
use serde_json::Value;
|
||||
use tokio::time::Instant;
|
||||
@@ -491,7 +492,9 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => into_http_response(response, ExpectedBody::SingleJson, "openai").await,
|
||||
Ok(response) => {
|
||||
into_http_response(response, ExpectedBody::SingleJson, OPENAI_PROTOCOL).await
|
||||
}
|
||||
Err(error) => error_response(error),
|
||||
}
|
||||
}
|
||||
@@ -513,7 +516,9 @@ pub async fn stream_execution_response(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => into_http_response(response, ExpectedBody::Sse, "openai").await,
|
||||
Ok(response) => {
|
||||
into_http_response(response, ExpectedBody::Sse, OPENAI_PROTOCOL).await
|
||||
}
|
||||
Err(error) => error_response(error),
|
||||
}
|
||||
},
|
||||
|
||||
@@ -15,6 +15,8 @@ use gateway_core::{
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
operation::OperationKind,
|
||||
};
|
||||
use gateway_protocol::anthropic::ANTHROPIC_PROTOCOL;
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum GenerationProtocol {
|
||||
@@ -27,7 +29,7 @@ impl GenerationProtocol {
|
||||
pub(crate) fn from_request(request: &DecodedGenerationRequest) -> Self {
|
||||
if request.operation().kind() == OperationKind::CountTokens {
|
||||
Self::MessageTokenCount
|
||||
} else if request.operation().protocol() == "anthropic" {
|
||||
} else if request.operation().protocol() == ANTHROPIC_PROTOCOL {
|
||||
Self::Messages
|
||||
} else {
|
||||
Self::Responses
|
||||
@@ -36,8 +38,8 @@ impl GenerationProtocol {
|
||||
|
||||
pub(crate) const fn wire(self) -> &'static str {
|
||||
match self {
|
||||
Self::Responses => "openai",
|
||||
Self::Messages | Self::MessageTokenCount => "anthropic",
|
||||
Self::Responses => OPENAI_PROTOCOL,
|
||||
Self::Messages | Self::MessageTokenCount => ANTHROPIC_PROTOCOL,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use axum::body::Bytes;
|
||||
use gateway_core::{error::GatewayError, event::ProviderEvent};
|
||||
use gateway_protocol::anthropic::ANTHROPIC_PROTOCOL;
|
||||
use gateway_protocol::{
|
||||
anthropic::MessagesAccumulator,
|
||||
openai::sse::{
|
||||
@@ -50,7 +51,7 @@ impl GenerationEncoder {
|
||||
} => {
|
||||
let Some(wire) = event
|
||||
.wire_event()
|
||||
.filter(|wire| wire.protocol() == "anthropic")
|
||||
.filter(|wire| wire.protocol() == ANTHROPIC_PROTOCOL)
|
||||
else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 出口校验按协议维护边界事实,不在流式路径保存生成正文
|
||||
|
||||
use gateway_core::event::ProviderEvent;
|
||||
use gateway_protocol::anthropic::ANTHROPIC_PROTOCOL;
|
||||
use gateway_protocol::openai::sse::parse_sse_events;
|
||||
use serde_json::Value;
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -32,7 +33,7 @@ impl GenerationValidationFacts {
|
||||
Self::Messages(id) => {
|
||||
if let Some(value) = event
|
||||
.wire_event()
|
||||
.filter(|wire| wire.protocol() == "anthropic")
|
||||
.filter(|wire| wire.protocol() == ANTHROPIC_PROTOCOL)
|
||||
.map(|wire| wire.data())
|
||||
&& let Some(value) = value
|
||||
.pointer("/message/id")
|
||||
|
||||
@@ -13,6 +13,7 @@ use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::operation::{
|
||||
ImageRequest, ImageRequestKind, Operation, OperationKind, RawJsonPayload,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ApiState;
|
||||
@@ -25,7 +26,6 @@ use crate::openai::{
|
||||
router::{IMAGE_EDITS_PATH, IMAGE_GENERATIONS_PATH},
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const IMAGE_TURN_ID_CONTEXT_KEY: &str = "image_turn_id";
|
||||
|
||||
/// `POST /v1/images/generations`
|
||||
|
||||
@@ -13,6 +13,7 @@ use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::event::ProviderEvent;
|
||||
use gateway_core::live::LiveHangupRequest;
|
||||
use gateway_core::operation::Operation;
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{
|
||||
@@ -31,9 +32,6 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
/// 引导 wire 事件使用的协议名;与 Provider 侧 `PROVIDER_NAME` 一致。
|
||||
const LIVE_PROTOCOL: &str = "openai";
|
||||
|
||||
pub(crate) async fn live_call(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
@@ -344,7 +342,7 @@ fn live_call_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut raw_events = 0usize;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == LIVE_PROTOCOL) else {
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == OPENAI_PROTOCOL) else {
|
||||
continue;
|
||||
};
|
||||
let raw = wire.into_raw_http_body()?;
|
||||
|
||||
@@ -20,6 +20,7 @@ use gateway_core::event::ProviderResponseHeader;
|
||||
use gateway_core::operation::{
|
||||
ProviderHttpHeader, ProviderHttpMethod, ProviderHttpRequest, RawHttpPayload,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::openai::error::openai_error_response;
|
||||
@@ -172,7 +173,7 @@ pub(crate) fn live_call_operation(
|
||||
"invalid_request",
|
||||
))
|
||||
};
|
||||
let payload = RawHttpPayload::new("openai", body).map_err(|_| invalid())?;
|
||||
let payload = RawHttpPayload::new(OPENAI_PROTOCOL, body).map_err(|_| invalid())?;
|
||||
ProviderHttpRequest::new(
|
||||
LIVE_CALLS_ENDPOINT,
|
||||
ProviderHttpMethod::Post,
|
||||
|
||||
@@ -20,6 +20,7 @@ use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
operation::OperationKind,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use std::{future::Future, sync::Arc};
|
||||
|
||||
use super::http::{ExpectedBody, buffered_response, error_response, into_http_response};
|
||||
@@ -38,7 +39,7 @@ impl RequestInput {
|
||||
pub(crate) fn query(endpoint: String, model_hint: Option<String>, headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
endpoint,
|
||||
protocol: "openai".to_owned(),
|
||||
protocol: OPENAI_PROTOCOL.to_owned(),
|
||||
operation: None,
|
||||
transport: ClientTransport::HttpJson,
|
||||
model_hint,
|
||||
|
||||
@@ -10,6 +10,7 @@ use gateway_core::{
|
||||
engine::execution::AuthenticatedClient,
|
||||
routing::{PublicModelDescriptor, PublicModelId, PublicModelProfile},
|
||||
};
|
||||
use gateway_protocol::anthropic::ANTHROPIC_PROTOCOL;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -52,7 +53,7 @@ pub(crate) async fn models(
|
||||
|
||||
let mut input = RequestInput::query(super::router::MODELS_PATH.to_owned(), None, &headers);
|
||||
if anthropic {
|
||||
input.protocol = "anthropic".to_owned();
|
||||
input.protocol = ANTHROPIC_PROTOCOL.to_owned();
|
||||
}
|
||||
let response = middleware::query_response(service.execution(), client, input, move |client| {
|
||||
models_response(state, client, query, anthropic)
|
||||
@@ -173,7 +174,7 @@ pub(crate) async fn model_detail(
|
||||
&headers,
|
||||
);
|
||||
if anthropic {
|
||||
input.protocol = "anthropic".to_owned();
|
||||
input.protocol = ANTHROPIC_PROTOCOL.to_owned();
|
||||
}
|
||||
let response = middleware::query_response(
|
||||
service.execution(),
|
||||
|
||||
@@ -6,6 +6,7 @@ use axum::http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_core::operation::{ClientTurn, GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
is_transport_managed_request_header,
|
||||
@@ -17,7 +18,6 @@ use crate::openai::generation::{
|
||||
ContinuationIntent, DecodedGenerationRequest, GenerationRequestMetadata,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::collections::BTreeMap;
|
||||
|
||||
use bytes::Bytes;
|
||||
use gateway_core::event::{GatewayEvent, ProtocolWireEvent, ProviderEvent};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use gateway_protocol::openai::sse::{
|
||||
encode_sse_event_with_metadata, response_failed_sse_data_from_error_event,
|
||||
};
|
||||
@@ -12,8 +13,6 @@ use serde_json::Value;
|
||||
use super::error::ResponseEncodeError;
|
||||
use crate::client_error::capacity_error_for_client;
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
/// OpenAI Responses wire 的转发器
|
||||
///
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::{
|
||||
};
|
||||
|
||||
use gateway_core::event::{GatewayEvent, ProtocolWireEvent, ProviderEvent};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use gateway_protocol::openai::sse::{parse_sse_events, sse_frame_is_done};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
@@ -44,7 +45,7 @@ impl ResponseValidationFacts {
|
||||
if let Some(wire) = event
|
||||
.middleware_origin_wire()
|
||||
.or_else(|| event.wire_event())
|
||||
.filter(|wire| wire.protocol() == "openai")
|
||||
.filter(|wire| wire.protocol() == OPENAI_PROTOCOL)
|
||||
{
|
||||
facts.observe_wire(wire);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ use gateway_core::engine::response_control::{ResponseControl, ResponseControlUna
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
use gateway_core::operation::ProviderSessionState;
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
|
||||
use crate::openai::error::{gateway_error_contract, gateway_error_from_engine};
|
||||
use crate::openai::middleware::{PendingExecution, pending_execution_response};
|
||||
@@ -118,7 +119,7 @@ pub(super) async fn execution_response(
|
||||
.collect();
|
||||
// 失败也要保留会话到洋葱链返回,否则释放会话会取消仍在等待 next 的插件
|
||||
return Ok(pending_execution_response(
|
||||
"openai".to_owned(),
|
||||
OPENAI_PROTOCOL.to_owned(),
|
||||
status,
|
||||
response_headers,
|
||||
MiddlewareFrame::new(
|
||||
@@ -142,7 +143,7 @@ pub(super) async fn execution_response(
|
||||
.map(|header| MiddlewareHeader::new(header.name(), header.value().clone()))
|
||||
.collect();
|
||||
Ok(MiddlewareResponse::new(
|
||||
"openai".to_owned(),
|
||||
OPENAI_PROTOCOL.to_owned(),
|
||||
200,
|
||||
headers,
|
||||
Box::new(body),
|
||||
@@ -159,7 +160,7 @@ pub(super) async fn forward_response(
|
||||
response_control: &ResponseControl,
|
||||
) -> ForwardOutcome {
|
||||
let (protocol, status, headers, mut body, _) = response.into_parts();
|
||||
if protocol != "openai" || StatusCode::from_u16(status).is_err() {
|
||||
if protocol != OPENAI_PROTOCOL || StatusCode::from_u16(status).is_err() {
|
||||
return fail_body(connection, body, MiddlewareError::InvalidState, &request_id).await;
|
||||
}
|
||||
let response_headers = headers
|
||||
|
||||
@@ -26,6 +26,7 @@ use gateway_core::{
|
||||
lifecycle::{ConnectionGuard, ConnectionLifecycle},
|
||||
operation::OperationKind,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
|
||||
use crate::middleware::headers::encode_headers;
|
||||
use crate::{
|
||||
@@ -328,7 +329,7 @@ async fn serve_responses_websocket(socket: Socket, session: ResponsesWebSocketSe
|
||||
let validation = ResponseValidationFacts::default();
|
||||
let input = RequestInput {
|
||||
endpoint: crate::openai::router::RESPONSES_PATH.to_owned(),
|
||||
protocol: "openai".to_owned(),
|
||||
protocol: OPENAI_PROTOCOL.to_owned(),
|
||||
operation: Some(OperationKind::Generate),
|
||||
transport: ClientTransport::WebSocket,
|
||||
model_hint,
|
||||
@@ -349,7 +350,7 @@ async fn serve_responses_websocket(socket: Socket, session: ResponsesWebSocketSe
|
||||
Box::new(move |prepared, request| {
|
||||
Box::pin(async move {
|
||||
let (protocol, headers, body) = request_parts(request.clone())?;
|
||||
if protocol != "openai" {
|
||||
if protocol != OPENAI_PROTOCOL {
|
||||
return Err(MiddlewareError::Rejected);
|
||||
}
|
||||
let payload =
|
||||
|
||||
@@ -176,6 +176,7 @@ pub(super) fn initial_engine_error_event(
|
||||
}
|
||||
let status = if provider
|
||||
.is_some_and(|error| error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable)
|
||||
&& gateway.client_error_code() != Some("previous_response_not_found")
|
||||
{
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
} else {
|
||||
|
||||
@@ -11,6 +11,7 @@ use axum::{
|
||||
use gateway_core::engine::execution::ClientTransport;
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::operation::{Operation, OperationKind, RawJsonPayload, StandaloneSearchRequest};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::middleware::headers::encode_headers;
|
||||
@@ -22,8 +23,6 @@ use crate::openai::{
|
||||
router::SEARCH_PATH,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
/// `POST /v1/alpha/search`
|
||||
pub(crate) async fn standalone_search(
|
||||
State(state): State<ApiState>,
|
||||
|
||||
@@ -15,6 +15,7 @@ use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLif
|
||||
use gateway_core::routing::{
|
||||
ProviderCatalogUnavailable, ProviderKind, PublicModelDescriptor, PublicModelId, UpstreamModelId,
|
||||
};
|
||||
use gateway_protocol::openai::OPENAI_PROTOCOL;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::auth::ClientApiKeyAuthError;
|
||||
@@ -159,7 +160,7 @@ impl OpenAiService {
|
||||
upstream_model,
|
||||
operation,
|
||||
ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
protocol: OPENAI_PROTOCOL.to_owned(),
|
||||
endpoint,
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
|
||||
@@ -413,7 +413,7 @@ fn account_page_item(
|
||||
protection_rules: Vec::new(),
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -580,7 +580,7 @@ mod provider {
|
||||
},
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -664,6 +664,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
.values
|
||||
.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: command.values.smart_scheduling,
|
||||
quota_reset_window_weights: command.values.quota_reset_window_weights,
|
||||
min_codex_desktop_version: command.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: command.values.min_codex_cli_version,
|
||||
usage_retention_days: command.values.usage_retention_days,
|
||||
@@ -1940,6 +1941,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -11,11 +11,11 @@ use chrono::Utc;
|
||||
use gateway_admin::{
|
||||
model::{MutationContext, Revision, proxies::*},
|
||||
ports::{
|
||||
proxy::{ProxyProbe, ProxyStore},
|
||||
proxy::{ProbeRoute, ProxyProbe, ProxyStore},
|
||||
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult},
|
||||
},
|
||||
};
|
||||
use gateway_core::account::{OutboundProxy, ProviderAccountId};
|
||||
use gateway_core::account::ProviderAccountId;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
@@ -348,17 +348,17 @@ pub(super) struct SuccessfulProbe;
|
||||
|
||||
#[async_trait]
|
||||
impl ProxyProbe for SuccessfulProbe {
|
||||
async fn test(&self, proxy: Option<&OutboundProxy>, detect_location: bool) -> ProxyTestResult {
|
||||
if let Some(proxy) = proxy {
|
||||
async fn test(&self, proxy: ProbeRoute<'_>, detect_location: bool) -> ProxyTestResult {
|
||||
if let ProbeRoute::Proxy(proxy) = proxy {
|
||||
assert_eq!(
|
||||
proxy.expose_url(),
|
||||
"http://test-user:private-password@proxy.example:8080/"
|
||||
);
|
||||
}
|
||||
let exit_ip = if proxy.is_some() {
|
||||
"203.0.113.2"
|
||||
} else {
|
||||
"203.0.113.3"
|
||||
let exit_ip = match proxy {
|
||||
ProbeRoute::Proxy(_) => "203.0.113.2",
|
||||
ProbeRoute::Direct => "203.0.113.3",
|
||||
ProbeRoute::Deployment => "203.0.113.4",
|
||||
};
|
||||
ProxyTestResult {
|
||||
location: if detect_location {
|
||||
@@ -600,6 +600,15 @@ async fn proxy_probe_checks_direct_egress_without_changing_saved_proxies() {
|
||||
}
|
||||
);
|
||||
}
|
||||
let (status, deployed) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/probe",
|
||||
Some(json!({"proxyUrl": "", "useDeploymentProxy": true})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(deployed["data"]["exitIpv4"], "203.0.113.4");
|
||||
let (_, listed) = request(&fixture, "/api/admin/proxies", None, true).await;
|
||||
assert_eq!(listed["data"]["page"]["total"], 1);
|
||||
assert_eq!(
|
||||
@@ -613,6 +622,10 @@ async fn proxy_probe_rejects_missing_or_invalid_addresses() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (body, expected) in [
|
||||
(
|
||||
json!({"proxyUrl": "http://proxy.example:8080", "useDeploymentProxy": true}),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(json!({"proxyUrl": " "}), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!({"proxyUrl": null}), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!({}), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
|
||||
@@ -63,6 +63,7 @@ fn update_body() -> Value {
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"quotaResetWindowWeights": gateway_core::account::ResetWindowWeights::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 31,
|
||||
@@ -264,12 +265,14 @@ fn error_protection_requires_an_explicit_policy_in_settings_replacement() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_intact() {
|
||||
async fn scheduling_settings_round_trip_and_invalid_updates_leave_the_saved_value_intact() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
let custom = json!({"loadWeight": 2.0, "quotaWeight": 0.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3});
|
||||
let custom = json!({"loadWeight": 2.0, "quotaWeight": 0.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3, "resetWindowWeights":{"fiveHour":0.0,"weekly":1.5,"monthly":0.0}});
|
||||
body["smartScheduling"] = custom.clone();
|
||||
let windows = json!({"fiveHour":2.5,"weekly":0.0,"monthly":1.0});
|
||||
body["quotaResetWindowWeights"] = windows.clone();
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
@@ -281,6 +284,7 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = response_json(response).await;
|
||||
assert_eq!(response["data"]["smartScheduling"], custom);
|
||||
assert_eq!(response["data"]["quotaResetWindowWeights"], windows);
|
||||
assert_eq!(
|
||||
response["data"]["smartSchedulingDefaults"],
|
||||
json!(gateway_core::account::SmartSchedulingConfig::default())
|
||||
@@ -301,36 +305,61 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
invalid_values.push(missing);
|
||||
}
|
||||
for invalid in invalid_values {
|
||||
body["smartScheduling"] = invalid;
|
||||
let invalid_windows = [
|
||||
json!({}),
|
||||
json!({"weekly":1}),
|
||||
json!(null),
|
||||
json!({"fiveHour":0,"weekly":1.23,"monthly":0}),
|
||||
json!({"fiveHour":-1,"weekly":1,"monthly":0}),
|
||||
json!({"fiveHour":0,"weekly":1,"monthly":10.1}),
|
||||
json!({"fiveHour":0,"weekly":1,"monthly":0,"daily":1}),
|
||||
];
|
||||
for windows in &invalid_windows {
|
||||
let mut invalid = custom.clone();
|
||||
invalid["resetWindowWeights"] = windows.clone();
|
||||
invalid_values.push(invalid);
|
||||
}
|
||||
for (field, invalid) in invalid_values
|
||||
.into_iter()
|
||||
.map(|value| ("smartScheduling", value))
|
||||
.chain(
|
||||
invalid_windows
|
||||
.into_iter()
|
||||
.map(|value| ("quotaResetWindowWeights", value)),
|
||||
)
|
||||
{
|
||||
let mut invalid_body = body.clone();
|
||||
invalid_body[field] = invalid;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body.clone()),
|
||||
Some(invalid_body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
for field in ["smartScheduling", "quotaResetWindowWeights"] {
|
||||
let mut missing = body.clone();
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(missing),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
body.as_object_mut().unwrap().remove("smartScheduling");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["smartScheduling"],
|
||||
custom
|
||||
);
|
||||
let saved = response_json(response).await;
|
||||
assert_eq!(saved["data"]["smartScheduling"], custom);
|
||||
assert_eq!(saved["data"]["quotaResetWindowWeights"], windows);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -344,11 +373,11 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it() {
|
||||
async fn request_retention_round_trips_and_invalid_days_do_not_replace_it() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
for days in [1, 3, 7, 15, 30, 31] {
|
||||
for days in [1, 31, 32, 365, 3650, u32::MAX] {
|
||||
body["usageRetentionDays"] = json!(days);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
@@ -365,8 +394,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it(
|
||||
}
|
||||
for (days, expected_status) in [
|
||||
(json!(0), StatusCode::BAD_REQUEST),
|
||||
(json!(32), StatusCode::BAD_REQUEST),
|
||||
(json!(u32::MAX), StatusCode::BAD_REQUEST),
|
||||
(json!(u64::from(u32::MAX) + 1), StatusCode::BAD_REQUEST),
|
||||
(json!(-1), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!(1.5), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!(null), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
@@ -389,7 +417,7 @@ async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it(
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["usageRetentionDays"],
|
||||
31
|
||||
u32::MAX
|
||||
);
|
||||
}
|
||||
|
||||
@@ -410,7 +438,12 @@ fn settings_request_validates_cron_and_allows_disabled_empty_schedule() {
|
||||
(false, "08:00", false),
|
||||
(true, "0 8 * * MON-FRI", true),
|
||||
(true, "60 8 * * *", false),
|
||||
(true, "0 8 * * *;30 13 * * *", true),
|
||||
(true, "0 8 * * *\n30 13 * * *", false),
|
||||
(true, "0 8 * * *;60 13 * * *", false),
|
||||
(false, "0 8 * * *;", false),
|
||||
(true, "0 8 * * *;;30 13 * * *", false),
|
||||
(true, "0 8 * * *\r\n30 13 * * *", false),
|
||||
] {
|
||||
let mut body = update_body();
|
||||
body["accountWarmupEnabled"] = json!(enabled);
|
||||
@@ -423,6 +456,11 @@ fn settings_request_validates_cron_and_allows_disabled_empty_schedule() {
|
||||
assert_eq!(error.field(), "accountWarmupCron");
|
||||
}
|
||||
}
|
||||
let mut oversized = update_body();
|
||||
oversized["accountWarmupCron"] = json!(["* * * * *"; 26].join(";"));
|
||||
let request: UpdateRuntimeSettingsRequest = serde_json::from_value(oversized).unwrap();
|
||||
assert_eq!(request.validate().unwrap_err().field(), "accountWarmupCron");
|
||||
|
||||
let mut legacy = update_body();
|
||||
legacy.as_object_mut().unwrap().remove("accountWarmupCron");
|
||||
legacy["accountWarmupScheduleTime"] = json!("08:00");
|
||||
|
||||
@@ -10,6 +10,14 @@ use std::time::SystemTime;
|
||||
pub(super) struct TestPeerInfo(pub Arc<Mutex<Option<ProviderQuota>>>);
|
||||
|
||||
impl UpstreamInfoSource for TestPeerInfo {
|
||||
fn is_peer(&self, _: &ProviderAccount) -> bool {
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.upstream.as_ref().is_some_and(|info| info.discovered))
|
||||
}
|
||||
|
||||
fn signal(
|
||||
&self,
|
||||
_: &ProviderAccount,
|
||||
|
||||
@@ -60,7 +60,8 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
for raw_response in [false, true] {
|
||||
let original = json!({"error": {
|
||||
"code": code, "type": "service_unavailable_error", "message": "busy",
|
||||
"param": "model", "future": {"keep": true}
|
||||
"param": "model", "future": {"keep": true},
|
||||
"headers": {"Retry-After":"309", "retry-after-ms":"309000", "x-request-id":"req_capacity"}
|
||||
}});
|
||||
let mut provider = ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
@@ -88,6 +89,10 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
Bytes::from_static(b"req_capacity"),
|
||||
),
|
||||
ProviderResponseHeader::new("retry-after", Bytes::from_static(b"7")),
|
||||
ProviderResponseHeader::new(
|
||||
"retry-after-ms",
|
||||
Bytes::from_static(b"7000"),
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
@@ -96,7 +101,8 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
if raw_response {
|
||||
assert_eq!(response.headers()["x-request-id"], "req_capacity");
|
||||
assert_eq!(response.headers()["retry-after"], "7");
|
||||
assert!(!response.headers().contains_key("retry-after"));
|
||||
assert!(!response.headers().contains_key("retry-after-ms"));
|
||||
}
|
||||
let body: Value = serde_json::from_slice(
|
||||
&to_bytes(response.into_body(), 4096)
|
||||
@@ -110,6 +116,7 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
if raw_response {
|
||||
let mut expected = original.clone();
|
||||
expected["error"]["code"] = json!("server_error");
|
||||
expected["error"]["headers"] = json!({"x-request-id":"req_capacity"});
|
||||
assert_eq!(body, expected);
|
||||
}
|
||||
let EngineError::Provider(provider) = error else {
|
||||
@@ -138,6 +145,38 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_client_replay_preserves_recovery_status_and_failure_correlation() {
|
||||
use bytes::Bytes;
|
||||
use gateway_core::error::ClientVisibleUpstreamResponse;
|
||||
use gateway_core::upstream::OpaqueUpstreamValue;
|
||||
use std::time::Duration;
|
||||
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_retry_after(Duration::from_secs(309))
|
||||
.with_upstream_request_id(OpaqueUpstreamValue::new("req_original_capacity"))
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Resend full history.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
400,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"code":"previous_response_not_found"}}"#),
|
||||
)),
|
||||
);
|
||||
let response = engine_error_response(&error);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert_eq!(response.headers()["x-request-id"], "req_original_capacity");
|
||||
assert!(!response.headers().contains_key("retry-after"));
|
||||
assert!(!response.headers().contains_key("retry-after-ms"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn classified_capacity_error_without_special_code_returns_retryable_http_status() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -4,6 +4,7 @@ use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
@@ -122,10 +123,12 @@ async fn native_json_preserves_extensions_and_uses_client_key_authentication() {
|
||||
)]);
|
||||
let trace = session.trace.clone();
|
||||
let (router, execution) = fixture(session).await;
|
||||
let response = router
|
||||
.oneshot(request("/v1/messages", false))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut incoming = request("/v1/messages", false);
|
||||
incoming.headers_mut().insert(
|
||||
"x-claude-code-session-id",
|
||||
"claude-session".parse().unwrap(),
|
||||
);
|
||||
let response = router.oneshot(incoming).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(bytes, raw);
|
||||
@@ -136,6 +139,15 @@ async fn native_json_preserves_extensions_and_uses_client_key_authentication() {
|
||||
panic!("generate")
|
||||
};
|
||||
assert_eq!(generation.protocol_payload().protocol(), "anthropic");
|
||||
assert!(
|
||||
generation.protocol_payload().context()["opaque_request_headers"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!([
|
||||
"x-claude-code-session-id",
|
||||
STANDARD.encode("claude-session")
|
||||
]))
|
||||
);
|
||||
assert_eq!(
|
||||
generation.protocol_payload().body()["future_extension"]["preserved"],
|
||||
true
|
||||
|
||||
@@ -99,7 +99,7 @@ async fn fixture(sharing: ClientPeerSharing, plaintext: &str) -> (AdminTestFixtu
|
||||
projection: resolve_account_status(&facts, now.into()),
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
id: "acct_private_sentinel".into(),
|
||||
provider_kind: provider,
|
||||
|
||||
@@ -2164,13 +2164,14 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_upstream_error_event_should_be_translated_to_response_failed_for_sse() {
|
||||
for (error_type, code, client_code, kind, client_status) in [
|
||||
for (error_type, code, client_code, kind, client_status, keeps_retry_after) in [
|
||||
(
|
||||
"service_unavailable_error",
|
||||
"server_is_overloaded",
|
||||
"server_error",
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
503,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"rate_limit_error",
|
||||
@@ -2178,13 +2179,14 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
"turn_rate_limit_exceeded",
|
||||
ProviderErrorKind::RateLimited,
|
||||
429,
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error_data = json!({
|
||||
"type": "error",
|
||||
"status": 429,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"headers": {"Retry-After": "24", "retry-after-ms": "24000", "x-request-id": "req_test"},
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": code,
|
||||
@@ -2254,6 +2256,31 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
.find(|event| event.event.as_deref() == Some("response.failed"))
|
||||
.expect("translated response.failed event");
|
||||
let failed: Value = serde_json::from_str(&failed.data).expect("response.failed JSON");
|
||||
let mut expected = json!({
|
||||
"type": "response.failed",
|
||||
"status": client_status,
|
||||
"headers": {"x-request-id": "req_test"},
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": client_code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
});
|
||||
if keeps_retry_after {
|
||||
expected["headers"]["Retry-After"] = json!("24");
|
||||
expected["headers"]["retry-after-ms"] = json!("24000");
|
||||
expected["response"]["error"]["headers"] = json!({"retry-after": "24"});
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -2262,32 +2289,8 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
body.contains("event: error\n"),
|
||||
body.ends_with("data: [DONE]\n\n"),
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"status": client_status,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": client_code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true},
|
||||
"headers": {"retry-after": "24"}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
}),
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
(expected, 1, false, true),
|
||||
"{code}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1004,7 +1004,8 @@ fn capacity_failure_projection_preserves_wire_metadata_and_original_events() {
|
||||
("response.failed", false),
|
||||
] {
|
||||
for status in [None, Some(400), Some(429), Some(503)] {
|
||||
let error = json!({"code": code, "type": "service_unavailable_error", "message": "busy", "future": 42});
|
||||
let error = json!({"code": code, "type": "service_unavailable_error", "message": "busy", "future": 42,
|
||||
"headers":{"retry-after":"309", "Retry-After-Ms":"309000", "x-request-id":"req_capacity"}});
|
||||
let mut original = if event_type == "error" {
|
||||
json!({"type": event_type, "error": error, "sequence_number": 3})
|
||||
} else {
|
||||
@@ -1046,11 +1047,16 @@ fn capacity_failure_projection_preserves_wire_metadata_and_original_events() {
|
||||
assert_eq!(sse_data["response"]["error"]["code"], "server_error");
|
||||
assert_eq!(sse_data["response"]["error"]["message"], "busy");
|
||||
assert_eq!(sse_data["response"]["error"]["future"], 42);
|
||||
assert_eq!(
|
||||
sse_data["response"]["error"]["headers"],
|
||||
json!({"x-request-id":"req_capacity"})
|
||||
);
|
||||
let messages = ws.push_websocket(&event);
|
||||
let ws_data: Value = serde_json::from_str(&messages[0]).expect("JSON");
|
||||
if event_type == "error" && status.is_some() {
|
||||
let mut expected = original.clone();
|
||||
expected["error"]["code"] = json!("server_error");
|
||||
expected["error"]["headers"] = json!({"x-request-id":"req_capacity"});
|
||||
expected["status_code"] = json!(503);
|
||||
assert_eq!(ws_data, expected);
|
||||
} else {
|
||||
|
||||
@@ -285,6 +285,7 @@ async fn initial_capacity_failure_offers_client_retry_with_upstream_correlation(
|
||||
vec![
|
||||
header("x-request-id", b"req_capacity"),
|
||||
header("retry-after", b"7"),
|
||||
header("retry-after-ms", b"7000"),
|
||||
],
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
@@ -297,7 +298,8 @@ async fn initial_capacity_failure_offers_client_retry_with_upstream_correlation(
|
||||
assert_eq!(error["error"]["code"], "server_error");
|
||||
assert_eq!(error["error"]["message"], "busy");
|
||||
assert_eq!(error["headers"]["x-request-id"], "req_capacity");
|
||||
assert_eq!(error["headers"]["retry-after"], "7");
|
||||
assert!(error["headers"].get("retry-after").is_none());
|
||||
assert!(error["headers"].get("retry-after-ms").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,20 +13,28 @@ pub use model_access::{
|
||||
AccountModelAccess, AccountModelAccessMode, InvalidAccountModelAccess,
|
||||
MAX_ACCOUNT_ACCESS_MODELS,
|
||||
};
|
||||
mod quota_limits;
|
||||
pub use quota_limits::{AccountQuotaLimits, QuotaMergeBase, QuotaPauseDecision};
|
||||
mod quota_settings;
|
||||
pub use quota_settings::{
|
||||
AccountQuotaSettings, QuotaBucketSettings, QuotaBucketWarmup, QuotaMergeBase,
|
||||
QuotaPauseDecision,
|
||||
};
|
||||
mod proxy;
|
||||
pub use proxy::{InvalidOutboundProxy, OutboundProxy};
|
||||
mod scheduling;
|
||||
pub mod scope;
|
||||
mod selection;
|
||||
mod smart_scheduling;
|
||||
mod store;
|
||||
mod upstream;
|
||||
|
||||
pub use error::CredentialError;
|
||||
pub use model::*;
|
||||
pub use scheduling::{
|
||||
QuotaResetPeriod, ResetWindowWeights, SchedulingConfigError, SmartSchedulingConfig,
|
||||
};
|
||||
pub(crate) use selection::smart_score;
|
||||
pub use selection::*;
|
||||
pub use smart_scheduling::{SmartSchedulingConfig, SmartSchedulingConfigError};
|
||||
pub use store::ProviderAccountStore;
|
||||
pub use store::{
|
||||
AccountCatalogObservation, AccountCatalogOmissions, ProviderAccountStore, QuotaWarmupState,
|
||||
QuotaWarmupStateUpdate, StoredAccountCatalog,
|
||||
};
|
||||
pub use upstream::*;
|
||||
|
||||
@@ -828,7 +828,7 @@ pub struct ProviderAccount {
|
||||
enabled: bool,
|
||||
concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
weight: AccountWeight,
|
||||
quota_limits: super::AccountQuotaLimits,
|
||||
quota_settings: super::AccountQuotaSettings,
|
||||
quota_paused: bool,
|
||||
auto_use_quota_credits: bool,
|
||||
model_access: super::AccountModelAccess,
|
||||
@@ -869,7 +869,7 @@ impl ProviderAccount {
|
||||
enabled: true,
|
||||
concurrency_limit: None,
|
||||
weight: AccountWeight::DEFAULT,
|
||||
quota_limits: super::AccountQuotaLimits::disabled(),
|
||||
quota_settings: super::AccountQuotaSettings::disabled(),
|
||||
quota_paused: false,
|
||||
auto_use_quota_credits: false,
|
||||
model_access: super::AccountModelAccess::all(),
|
||||
@@ -886,16 +886,20 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_quota_limits(mut self, limits: super::AccountQuotaLimits, paused: bool) -> Self {
|
||||
pub fn with_quota_settings(
|
||||
mut self,
|
||||
limits: super::AccountQuotaSettings,
|
||||
paused: bool,
|
||||
) -> Self {
|
||||
// 只有限额账号需要在取得租约前检测额度窗口的并发更新
|
||||
self.quota_paused = paused;
|
||||
self.quota_limits = limits;
|
||||
self.quota_settings = limits;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota_limits(&self) -> &super::AccountQuotaLimits {
|
||||
&self.quota_limits
|
||||
pub const fn quota_settings(&self) -> &super::AccountQuotaSettings {
|
||||
&self.quota_settings
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
use std::fmt;
|
||||
use url::Url;
|
||||
|
||||
use crate::upstream::ProxyEndpoint;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct OutboundProxy {
|
||||
url: Url,
|
||||
@@ -51,6 +53,26 @@ impl OutboundProxy {
|
||||
self.url.as_str()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn dial_endpoint(&self) -> ProxyEndpoint {
|
||||
ProxyEndpoint::new(
|
||||
self.url.host_str().expect("validated proxy host"),
|
||||
self.url
|
||||
.port_or_known_default()
|
||||
.expect("validated proxy port"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn matches_endpoint(&self, endpoint: &ProxyEndpoint) -> bool {
|
||||
endpoint.matches(
|
||||
self.url.host_str().expect("validated proxy host"),
|
||||
self.url
|
||||
.port_or_known_default()
|
||||
.expect("validated proxy port"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn endpoint(&self) -> String {
|
||||
let mut url = self.url.clone();
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
//! 管理员按账号通用窗口设置的已用比例上限
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(try_from = "BTreeMap<String, u8>")]
|
||||
pub struct AccountQuotaLimits(BTreeMap<String, u8>);
|
||||
|
||||
impl AccountQuotaLimits {
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self(BTreeMap::new())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.0.contains_key(key)
|
||||
}
|
||||
|
||||
pub fn keys(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.keys().map(String::as_str)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reached(&self, key: &str, used: Option<f64>) -> bool {
|
||||
self.0
|
||||
.get(key)
|
||||
.zip(used)
|
||||
.is_some_and(|(&limit, used)| used >= f64::from(limit))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<BTreeMap<String, u8>> for AccountQuotaLimits {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from(limits: BTreeMap<String, u8>) -> Result<Self, Self::Error> {
|
||||
if limits.iter().any(|(key, limit)| {
|
||||
key.is_empty()
|
||||
|| key.len() > 256
|
||||
|| key.chars().any(char::is_control)
|
||||
|| !(1..=100).contains(limit)
|
||||
}) {
|
||||
return Err("额度窗口标识无效或上限不在 1~100 范围内");
|
||||
}
|
||||
Ok(Self(limits))
|
||||
}
|
||||
}
|
||||
|
||||
/// 合并时读到的快照版本,时间为空表示当时尚无快照
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaMergeBase {
|
||||
pub observed_at: Option<std::time::SystemTime>,
|
||||
}
|
||||
|
||||
/// 此次判断依据的策略,以及是否有通用窗口触发暂停
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct QuotaPauseDecision {
|
||||
/// 独立观测无需合并,依赖旧快照的判断必须随写入校验版本
|
||||
pub merge_base: Option<QuotaMergeBase>,
|
||||
pub limits: AccountQuotaLimits,
|
||||
pub paused: bool,
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! 管理员按额度窗口设置的使用上限与到期预热策略
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(try_from = "BTreeMap<String, QuotaBucketSettings>")]
|
||||
pub struct AccountQuotaSettings(BTreeMap<String, QuotaBucketSettings>);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct QuotaBucketSettings {
|
||||
pub limit_percent: Option<u8>,
|
||||
pub warmup: Option<QuotaBucketWarmup>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct QuotaBucketWarmup {
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl AccountQuotaSettings {
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self(BTreeMap::new())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = (&str, &QuotaBucketSettings)> {
|
||||
self.0
|
||||
.iter()
|
||||
.map(|(key, settings)| (key.as_str(), settings))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_warmup(&self) -> bool {
|
||||
self.0.values().any(|settings| settings.warmup.is_some())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.0.contains_key(key)
|
||||
}
|
||||
|
||||
pub fn keys(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.keys().map(String::as_str)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reached(&self, key: &str, used: Option<f64>) -> bool {
|
||||
self.0
|
||||
.get(key)
|
||||
.and_then(|settings| settings.limit_percent)
|
||||
.zip(used)
|
||||
.is_some_and(|(limit, used)| used >= f64::from(limit))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<BTreeMap<String, QuotaBucketSettings>> for AccountQuotaSettings {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from(limits: BTreeMap<String, QuotaBucketSettings>) -> Result<Self, Self::Error> {
|
||||
if limits.iter().any(|(key, settings)| {
|
||||
key.is_empty()
|
||||
|| key.len() > 256
|
||||
|| key.chars().any(char::is_control)
|
||||
|| settings
|
||||
.limit_percent
|
||||
.is_some_and(|limit| !(1..=100).contains(&limit))
|
||||
|| settings.warmup.as_ref().is_some_and(|warmup| {
|
||||
warmup.model.is_empty()
|
||||
|| warmup.model.len() > 128
|
||||
|| warmup.model.trim() != warmup.model
|
||||
|| warmup.model.chars().any(char::is_control)
|
||||
})
|
||||
}) {
|
||||
return Err("额度桶设置无效");
|
||||
}
|
||||
Ok(Self(limits))
|
||||
}
|
||||
}
|
||||
|
||||
/// 合并时读到的快照版本,时间为空表示当时尚无快照
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaMergeBase {
|
||||
pub observed_at: Option<std::time::SystemTime>,
|
||||
}
|
||||
|
||||
/// 此次判断依据的策略,以及是否有通用窗口触发暂停
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct QuotaPauseDecision {
|
||||
/// 独立观测无需合并,依赖旧快照的判断必须随写入校验版本
|
||||
pub merge_base: Option<QuotaMergeBase>,
|
||||
pub limits: AccountQuotaSettings,
|
||||
pub paused: bool,
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
//! 账号调度参数与共享重置窗口评分;固定十分位保证快照可精确比较,浮点数只用于 wire 与评分
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
use super::AccountRuntimeSignals;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(into = "SmartSchedulingValues")]
|
||||
pub struct SmartSchedulingConfig {
|
||||
weights: [u8; 6],
|
||||
reset_windows: ResetWindowWeights,
|
||||
}
|
||||
|
||||
/// 账号通用额度的周期;Provider 按上游周期事实投影,不依赖窗口名称和槽位
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum QuotaResetPeriod {
|
||||
FiveHour,
|
||||
Weekly,
|
||||
Monthly,
|
||||
}
|
||||
|
||||
impl QuotaResetPeriod {
|
||||
const fn seconds(self) -> u32 {
|
||||
match self {
|
||||
Self::FiveHour => 18_000,
|
||||
Self::Weekly => 604_800,
|
||||
Self::Monthly => 2_592_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn weight_tenths(value: f64, field: &'static str) -> Result<u8, SchedulingConfigError> {
|
||||
if !(0.0..=10.0).contains(&value) || (value * 10.0).round() / 10.0 != value {
|
||||
return Err(SchedulingConfigError::InvalidWeight(field));
|
||||
}
|
||||
Ok((value * 10.0).round() as u8)
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SmartSchedulingConfig {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// flatten 会缓冲字段,arbitrary_precision 的小数需先恢复为 JSON Number
|
||||
// 随后仍由同一类型检查完整字段、数值范围和十分位精度
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let values = serde_json::from_value::<SmartSchedulingValues>(value)
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
Self::try_from(values).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SmartSchedulingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
weights: [10, 8, 10, 5, 0, 0],
|
||||
reset_windows: ResetWindowWeights::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum SchedulingConfigError {
|
||||
#[error("{0} must be between 0 and 10 with at most one decimal place")]
|
||||
InvalidWeight(&'static str),
|
||||
#[error("at least one smart scheduling weight must be positive")]
|
||||
EmptyWeights,
|
||||
}
|
||||
|
||||
impl SmartSchedulingConfig {
|
||||
/// 顺序为负载、剩余额度、健康、首输出延迟、额度重置和排队压力
|
||||
pub fn new(weights: [f64; 6]) -> Result<Self, SchedulingConfigError> {
|
||||
let mut tenths = [0; 6];
|
||||
for (index, field) in [
|
||||
"loadWeight",
|
||||
"quotaWeight",
|
||||
"healthWeight",
|
||||
"latencyWeight",
|
||||
"resetWeight",
|
||||
"queueWeight",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
tenths[index] = weight_tenths(weights[index], field)?;
|
||||
}
|
||||
if tenths == [0; 6] {
|
||||
return Err(SchedulingConfigError::EmptyWeights);
|
||||
}
|
||||
Ok(Self {
|
||||
weights: tenths,
|
||||
..Self::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn weights(self) -> [f64; 6] {
|
||||
self.weights.map(|weight| f64::from(weight) / 10.0)
|
||||
}
|
||||
|
||||
fn reset_enabled(self) -> bool {
|
||||
self.weights[4] > 0 && self.reset_windows.enabled()
|
||||
}
|
||||
|
||||
pub(crate) fn reset_score(self, signals: &AccountRuntimeSignals, now: SystemTime) -> f64 {
|
||||
if !self.reset_enabled() {
|
||||
return 0.0;
|
||||
}
|
||||
self.reset_windows.score(signals, now)
|
||||
}
|
||||
|
||||
pub(crate) fn score_tolerance(self) -> f64 {
|
||||
// 与系数同步缩放,避免仅放大相同比例的系数就改变近似最优候选集合
|
||||
// 排队和未启用的重置窗口不能放大立即选号的容差
|
||||
let count = if self.reset_enabled() { 5 } else { 4 };
|
||||
0.05 * f64::from(
|
||||
self.weights[..count]
|
||||
.iter()
|
||||
.copied()
|
||||
.map(u16::from)
|
||||
.sum::<u16>(),
|
||||
) / 33.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct SmartSchedulingValues {
|
||||
load_weight: f64,
|
||||
quota_weight: f64,
|
||||
health_weight: f64,
|
||||
latency_weight: f64,
|
||||
reset_weight: f64,
|
||||
queue_weight: f64,
|
||||
#[serde(default)]
|
||||
reset_window_weights: ResetWindowWeights,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct ResetWindowValues {
|
||||
five_hour: f64,
|
||||
weekly: f64,
|
||||
monthly: f64,
|
||||
}
|
||||
|
||||
impl TryFrom<SmartSchedulingValues> for SmartSchedulingConfig {
|
||||
type Error = SchedulingConfigError;
|
||||
|
||||
fn try_from(value: SmartSchedulingValues) -> Result<Self, Self::Error> {
|
||||
let mut config = Self::new([
|
||||
value.load_weight,
|
||||
value.quota_weight,
|
||||
value.health_weight,
|
||||
value.latency_weight,
|
||||
value.reset_weight,
|
||||
value.queue_weight,
|
||||
])?;
|
||||
config.reset_windows = value.reset_window_weights;
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SmartSchedulingConfig> for SmartSchedulingValues {
|
||||
fn from(value: SmartSchedulingConfig) -> Self {
|
||||
let [
|
||||
load_weight,
|
||||
quota_weight,
|
||||
health_weight,
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
] = value.weights();
|
||||
Self {
|
||||
load_weight,
|
||||
quota_weight,
|
||||
health_weight,
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
reset_window_weights: value.reset_windows,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 两种调度策略共用的重置窗口系数,各策略独立持有配置
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(into = "ResetWindowValues")]
|
||||
pub struct ResetWindowWeights {
|
||||
weights: [u8; 3],
|
||||
}
|
||||
|
||||
impl ResetWindowWeights {
|
||||
/// 顺序为 5 小时、周、月;全零表示未指定窗口
|
||||
pub fn new(weights: [f64; 3]) -> Result<Self, SchedulingConfigError> {
|
||||
Ok(Self {
|
||||
weights: [
|
||||
weight_tenths(weights[0], "fiveHour")?,
|
||||
weight_tenths(weights[1], "weekly")?,
|
||||
weight_tenths(weights[2], "monthly")?,
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn enabled(self) -> bool {
|
||||
self.weights != [0; 3]
|
||||
}
|
||||
|
||||
pub(crate) fn score(self, signals: &AccountRuntimeSignals, now: SystemTime) -> f64 {
|
||||
if !self.enabled() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut scores = [0.0_f64; 3];
|
||||
for window in signals.quota_reset_windows.as_deref().unwrap_or_default() {
|
||||
let Ok(remaining) = window.reset_at.duration_since(now) else {
|
||||
continue;
|
||||
};
|
||||
if remaining.is_zero() {
|
||||
continue;
|
||||
}
|
||||
let half = f64::from(window.period.seconds()) / 7.0;
|
||||
let score = half / (half + remaining.as_secs_f64());
|
||||
// 同一周期只计最近的有效重置,避免重复额度桶叠加得分
|
||||
let slot = &mut scores[window.period as usize];
|
||||
*slot = slot.max(score);
|
||||
}
|
||||
// 缺失周期得零分,分母固定,避免少报窗口反而提高排名
|
||||
let weights = self.weights.map(f64::from);
|
||||
scores
|
||||
.into_iter()
|
||||
.zip(weights)
|
||||
.map(|(score, w)| score * w)
|
||||
.sum::<f64>()
|
||||
/ weights.into_iter().sum::<f64>()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for ResetWindowWeights {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// 与智能调度一致,先恢复 flatten 缓冲的小数再校验
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let values =
|
||||
serde_json::from_value::<ResetWindowValues>(value).map_err(serde::de::Error::custom)?;
|
||||
Self::new([values.five_hour, values.weekly, values.monthly])
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ResetWindowWeights> for ResetWindowValues {
|
||||
fn from(value: ResetWindowWeights) -> Self {
|
||||
let [five_hour, weekly, monthly] = value.weights.map(|weight| f64::from(weight) / 10.0);
|
||||
Self {
|
||||
five_hour,
|
||||
weekly,
|
||||
monthly,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Mutex, RwLock};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use futures::FutureExt;
|
||||
@@ -12,7 +12,8 @@ use crate::concurrency::{CapacityWait, ConcurrencyQueuePolicy, QueueRejection};
|
||||
use crate::identity::ProviderKind;
|
||||
|
||||
use super::{
|
||||
AccountConcurrency, AccountStatus, ProviderAccount, ProviderAccountId, SmartSchedulingConfig,
|
||||
AccountConcurrency, AccountStatus, ProviderAccount, ProviderAccountId, ResetWindowWeights,
|
||||
SmartSchedulingConfig,
|
||||
};
|
||||
|
||||
/// `runtime_settings.rotation_strategy` 的稳定值
|
||||
@@ -90,6 +91,7 @@ impl AccountAffinity {
|
||||
pub struct AccountSelectionPolicy {
|
||||
strategy: RotationStrategy,
|
||||
smart_scheduling: SmartSchedulingConfig,
|
||||
quota_reset_window_weights: ResetWindowWeights,
|
||||
max_concurrent_per_account: AccountConcurrency,
|
||||
queue_policy: ConcurrencyQueuePolicy,
|
||||
openai_guardian_reserved_concurrency: u32,
|
||||
@@ -108,6 +110,7 @@ impl AccountSelectionPolicy {
|
||||
Self {
|
||||
strategy,
|
||||
smart_scheduling: SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: ResetWindowWeights::default(),
|
||||
max_concurrent_per_account: max_concurrent_per_account.into(),
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: AccountAffinity::default(),
|
||||
@@ -121,6 +124,17 @@ impl AccountSelectionPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_quota_reset_window_weights(mut self, weights: ResetWindowWeights) -> Self {
|
||||
self.quota_reset_window_weights = weights;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota_reset_window_weights(self) -> ResetWindowWeights {
|
||||
self.quota_reset_window_weights
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_account_affinity(mut self, affinity: AccountAffinity) -> Self {
|
||||
self.openai_account_affinity = affinity;
|
||||
@@ -245,6 +259,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub in_flight: u32,
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_reset_windows: Option<Arc<[AccountQuotaResetWindow]>>,
|
||||
/// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub cooldown: Option<super::AccountCooldown>,
|
||||
@@ -459,6 +474,7 @@ impl AccountRuntimeSignals {
|
||||
pub fn with_provider_quota(mut self, quota: Option<AccountQuotaSignals>) -> Self {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_reset_windows = quota.reset_windows;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
}
|
||||
self
|
||||
@@ -482,9 +498,17 @@ impl AccountRuntimeSignals {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 确认为账号通用且属于受支持周期的重置观测
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountQuotaResetWindow {
|
||||
pub period: super::QuotaResetPeriod,
|
||||
pub reset_at: SystemTime,
|
||||
}
|
||||
|
||||
/// Provider 从私有 quota JSON 投影出的中立调度事实
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_windows: Option<Arc<[AccountQuotaResetWindow]>>,
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
}
|
||||
@@ -496,16 +520,23 @@ impl AccountQuotaSignals {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
reset_windows: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reset_at(self) -> Option<SystemTime> {
|
||||
pub fn with_reset_windows(mut self, windows: Vec<AccountQuotaResetWindow>) -> Self {
|
||||
self.reset_windows = (!windows.is_empty()).then(|| windows.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reset_at(&self) -> Option<SystemTime> {
|
||||
self.reset_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
pub const fn remaining_rank(&self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
}
|
||||
@@ -782,24 +813,31 @@ impl AccountSelector {
|
||||
let candidate = match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
eligible.sort_by(|left, right| {
|
||||
(
|
||||
left.signals.quota_reset_at.is_none(),
|
||||
left.signals.quota_reset_at,
|
||||
)
|
||||
.cmp(&(
|
||||
right.signals.quota_reset_at.is_none(),
|
||||
right.signals.quota_reset_at,
|
||||
))
|
||||
.then_with(|| {
|
||||
capacity_utilization(left, context)
|
||||
.total_cmp(&capacity_utilization(right, context))
|
||||
})
|
||||
.then_with(|| {
|
||||
left.signals
|
||||
.last_started_at
|
||||
.cmp(&right.signals.last_started_at)
|
||||
})
|
||||
.then_with(|| left.account.id().cmp(right.account.id()))
|
||||
let windows = context.policy.quota_reset_window_weights();
|
||||
if windows.enabled() {
|
||||
windows
|
||||
.score(&right.signals, context.now)
|
||||
.total_cmp(&windows.score(&left.signals, context.now))
|
||||
} else {
|
||||
(
|
||||
left.signals.quota_reset_at.is_none(),
|
||||
left.signals.quota_reset_at,
|
||||
)
|
||||
.cmp(&(
|
||||
right.signals.quota_reset_at.is_none(),
|
||||
right.signals.quota_reset_at,
|
||||
))
|
||||
}
|
||||
.then_with(|| {
|
||||
capacity_utilization(left, context)
|
||||
.total_cmp(&capacity_utilization(right, context))
|
||||
})
|
||||
.then_with(|| {
|
||||
left.signals
|
||||
.last_started_at
|
||||
.cmp(&right.signals.last_started_at)
|
||||
})
|
||||
.then_with(|| left.account.id().cmp(right.account.id()))
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
@@ -940,8 +978,6 @@ impl AccountSelector {
|
||||
|
||||
// 首输出 10 秒时延迟得分减半;固定尺度不随其他候选账号变化
|
||||
const SMART_LATENCY_HALF_SCORE_MS: f64 = 10_000.0;
|
||||
// 距重置一小时时得分减半;未知和已过期时间不提供重置奖励
|
||||
const SMART_RESET_HALF_SCORE_SECONDS: f64 = 3_600.0;
|
||||
|
||||
fn capacity_utilization(candidate: &AccountCandidate, context: &AccountSelectionContext) -> f64 {
|
||||
context
|
||||
@@ -997,15 +1033,6 @@ pub(crate) fn smart_score(candidate: &AccountCandidate, context: &AccountSelecti
|
||||
SMART_LATENCY_HALF_SCORE_MS / (SMART_LATENCY_HALF_SCORE_MS + latency as f64)
|
||||
});
|
||||
|
||||
let reset = candidate
|
||||
.signals
|
||||
.quota_reset_at
|
||||
.and_then(|reset| reset.duration_since(now).ok())
|
||||
.filter(|remaining| !remaining.is_zero())
|
||||
.map_or(0.0, |remaining| {
|
||||
SMART_RESET_HALF_SCORE_SECONDS
|
||||
/ (SMART_RESET_HALF_SCORE_SECONDS + remaining.as_secs_f64())
|
||||
});
|
||||
let [
|
||||
load_weight,
|
||||
quota_weight,
|
||||
@@ -1018,5 +1045,5 @@ pub(crate) fn smart_score(candidate: &AccountCandidate, context: &AccountSelecti
|
||||
+ quota_weight * quota
|
||||
+ health_weight * failure
|
||||
+ latency_weight * latency
|
||||
+ reset_weight * reset
|
||||
+ reset_weight * config.reset_score(&candidate.signals, now)
|
||||
}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
//! 智能调度参数;固定十分位保证快照可精确比较,浮点数只用于 wire 与评分
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(into = "SmartSchedulingValues")]
|
||||
pub struct SmartSchedulingConfig {
|
||||
weights: [u8; 6],
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SmartSchedulingConfig {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// flatten 会缓冲字段,arbitrary_precision 的小数需先恢复为 JSON Number
|
||||
// 随后仍由同一类型检查完整字段、数值范围和十分位精度
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let values = serde_json::from_value::<SmartSchedulingValues>(value)
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
Self::try_from(values).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SmartSchedulingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
weights: [10, 8, 10, 5, 0, 0],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum SmartSchedulingConfigError {
|
||||
#[error("{0} must be between 0 and 10 with at most one decimal place")]
|
||||
InvalidWeight(&'static str),
|
||||
#[error("at least one smart scheduling weight must be positive")]
|
||||
EmptyWeights,
|
||||
}
|
||||
|
||||
impl SmartSchedulingConfig {
|
||||
/// 顺序为负载、剩余额度、健康、首输出延迟、额度重置和排队压力
|
||||
pub fn new(weights: [f64; 6]) -> Result<Self, SmartSchedulingConfigError> {
|
||||
let mut tenths = [0; 6];
|
||||
for (index, field) in [
|
||||
"loadWeight",
|
||||
"quotaWeight",
|
||||
"healthWeight",
|
||||
"latencyWeight",
|
||||
"resetWeight",
|
||||
"queueWeight",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let value = weights[index];
|
||||
if !value.is_finite()
|
||||
|| !(0.0..=10.0).contains(&value)
|
||||
|| (value * 10.0).round() / 10.0 != value
|
||||
{
|
||||
return Err(SmartSchedulingConfigError::InvalidWeight(field));
|
||||
}
|
||||
tenths[index] = (value * 10.0).round() as u8;
|
||||
}
|
||||
if tenths == [0; 6] {
|
||||
return Err(SmartSchedulingConfigError::EmptyWeights);
|
||||
}
|
||||
Ok(Self { weights: tenths })
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn weights(self) -> [f64; 6] {
|
||||
self.weights.map(|weight| f64::from(weight) / 10.0)
|
||||
}
|
||||
|
||||
pub(crate) fn score_tolerance(self) -> f64 {
|
||||
// 与系数同步缩放,避免仅放大相同比例的系数就改变近似最优候选集合
|
||||
// 排队系数只参与入队选择,不能放大立即选号的候选容差
|
||||
0.05 * f64::from(
|
||||
self.weights[..5]
|
||||
.iter()
|
||||
.copied()
|
||||
.map(u16::from)
|
||||
.sum::<u16>(),
|
||||
) / 33.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct SmartSchedulingValues {
|
||||
load_weight: f64,
|
||||
quota_weight: f64,
|
||||
health_weight: f64,
|
||||
latency_weight: f64,
|
||||
reset_weight: f64,
|
||||
queue_weight: f64,
|
||||
}
|
||||
|
||||
impl TryFrom<SmartSchedulingValues> for SmartSchedulingConfig {
|
||||
type Error = SmartSchedulingConfigError;
|
||||
|
||||
fn try_from(value: SmartSchedulingValues) -> Result<Self, Self::Error> {
|
||||
Self::new([
|
||||
value.load_weight,
|
||||
value.quota_weight,
|
||||
value.health_weight,
|
||||
value.latency_weight,
|
||||
value.reset_weight,
|
||||
value.queue_weight,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SmartSchedulingConfig> for SmartSchedulingValues {
|
||||
fn from(value: SmartSchedulingConfig) -> Self {
|
||||
let [
|
||||
load_weight,
|
||||
quota_weight,
|
||||
health_weight,
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
] = value.weights();
|
||||
Self {
|
||||
load_weight,
|
||||
quota_weight,
|
||||
health_weight,
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,9 +12,73 @@ use super::{
|
||||
QuotaObservationTouch, QuotaObservationVersion, QuotaWriteOutcome,
|
||||
};
|
||||
|
||||
/// 一个账号最后成功的目录观测;正文由 Provider 解释,归属不能按相同正文合并
|
||||
#[derive(Clone)]
|
||||
pub struct AccountCatalogObservation {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub revision: CredentialRevision,
|
||||
pub plan: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub observed_at: std::time::SystemTime,
|
||||
}
|
||||
|
||||
/// 批量恢复时每份不同的正文只传输、解析一次
|
||||
pub struct StoredAccountCatalog {
|
||||
pub accounts: Vec<AccountCatalogObservation>,
|
||||
pub document: bytes::Bytes,
|
||||
}
|
||||
|
||||
/// 批量选号只按账号当前凭据和套餐的目录摘要排除候选
|
||||
pub struct AccountCatalogOmissions {
|
||||
pub documents: Vec<[u8; 32]>,
|
||||
pub exceptions: Vec<ProviderAccountId>,
|
||||
pub as_of: std::time::SystemTime,
|
||||
}
|
||||
|
||||
/// Provider 自行解释预热进度;版本只用于并发领取和观测合并
|
||||
#[derive(Clone, Default)]
|
||||
pub struct QuotaWarmupState {
|
||||
pub version: u64,
|
||||
pub data: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
pub struct QuotaWarmupStateUpdate<'a> {
|
||||
pub account: &'a ProviderAccount,
|
||||
pub expected_version: u64,
|
||||
pub expected_quota_observed_at: Option<std::time::SystemTime>,
|
||||
pub data: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口
|
||||
#[async_trait]
|
||||
pub trait ProviderAccountStore: Send + Sync {
|
||||
async fn load_quota_warmup_state(
|
||||
&self,
|
||||
_account: &ProviderAccount,
|
||||
) -> Result<Option<QuotaWarmupState>, StoreError> {
|
||||
Err(StoreError::new(crate::error::StoreErrorKind::InvalidData))
|
||||
}
|
||||
|
||||
/// 配置、凭据或进度变化均使领取失败,取消或崩溃后不能重放已领取的请求
|
||||
async fn compare_and_swap_quota_warmup_state(
|
||||
&self,
|
||||
_update: QuotaWarmupStateUpdate<'_>,
|
||||
) -> Result<bool, StoreError> {
|
||||
Err(StoreError::new(crate::error::StoreErrorKind::InvalidData))
|
||||
}
|
||||
|
||||
async fn load_account_catalogs(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> Result<Vec<StoredAccountCatalog>, StoreError>;
|
||||
|
||||
/// 账号删除或凭据已变化时返回 false,不发布迟到的观测
|
||||
async fn save_account_catalog(
|
||||
&self,
|
||||
observation: &AccountCatalogObservation,
|
||||
document: &[u8],
|
||||
) -> Result<bool, StoreError>;
|
||||
|
||||
async fn create_account(&self, account: NewProviderAccount) -> Result<(), StoreError>;
|
||||
|
||||
async fn get_account(
|
||||
@@ -36,6 +100,16 @@ pub trait ProviderAccountStore: Send + Sync {
|
||||
account_ids: &[ProviderAccountId],
|
||||
) -> Result<Vec<ProviderAccount>, StoreError>;
|
||||
|
||||
/// 存储可以下推目录筛选,也可以返回完整候选;调用方始终复核最终账号
|
||||
async fn list_catalog_candidates(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
accounts: &[ProviderAccountId],
|
||||
_omissions: &AccountCatalogOmissions,
|
||||
) -> Result<Vec<ProviderAccount>, StoreError> {
|
||||
self.list_for_provider_accounts(provider, accounts).await
|
||||
}
|
||||
|
||||
/// 一次有界查询返回账号事实和 revision-fenced 明文 credential
|
||||
async fn list_refresh_candidates(
|
||||
&self,
|
||||
|
||||
@@ -84,6 +84,9 @@ pub struct UpstreamSchedulingSignal {
|
||||
}
|
||||
|
||||
pub trait UpstreamInfoSource: Send + Sync {
|
||||
/// 当前连接是否已识别为互联节点,不以额度快照的新鲜度代替身份
|
||||
fn is_peer(&self, account: &ProviderAccount) -> bool;
|
||||
|
||||
fn signal(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
@@ -110,6 +113,13 @@ impl UpstreamInfoReader {
|
||||
}
|
||||
|
||||
impl UpstreamInfoSource for UpstreamInfoReader {
|
||||
fn is_peer(&self, account: &ProviderAccount) -> bool {
|
||||
self.source
|
||||
.get()
|
||||
.and_then(Weak::upgrade)
|
||||
.is_some_and(|source| source.is_peer(account))
|
||||
}
|
||||
|
||||
fn signal(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
|
||||
@@ -158,6 +158,7 @@ impl TraceContext {
|
||||
"recovery": {
|
||||
"replaySafe": error.replay_is_safe(),
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sameAccountRetryBlocked": error.same_account_retry_blocked(),
|
||||
"sameAccountCredentialRetry": error.retries_same_account(),
|
||||
"intent": error.pre_delivery_retry().map(|intent| format!("{intent:?}")),
|
||||
"continuationFailure": error.continuation_failure().map(|failure| format!("{failure:?}")),
|
||||
@@ -346,6 +347,7 @@ impl TraceContext {
|
||||
for key in [
|
||||
"replaySafe",
|
||||
"retryProhibited",
|
||||
"sameAccountRetryBlocked",
|
||||
"sameAccountCredentialRetry",
|
||||
] {
|
||||
kept[key] = json!(recovery.get(key).and_then(Value::as_bool));
|
||||
|
||||
@@ -367,6 +367,7 @@ where
|
||||
account_rotations: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
pending_retry: None,
|
||||
@@ -525,6 +526,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
account_rotations: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
excluded_proxy_endpoints: Vec<crate::upstream::ProxyEndpoint>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算
|
||||
@@ -1019,6 +1021,10 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn has_account_rotation_budget(&self) -> bool {
|
||||
self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
let pending_retry = self.pending_retry.take();
|
||||
let is_transport_recovery = pending_retry
|
||||
@@ -1094,7 +1100,7 @@ where
|
||||
),
|
||||
}
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
let mut account_context = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(target) => AccountAttemptContext::diagnostic(
|
||||
self.excluded_accounts.clone(),
|
||||
target.clone(),
|
||||
@@ -1111,6 +1117,9 @@ where
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
}));
|
||||
account_context
|
||||
.excluded_proxy_endpoints
|
||||
.clone_from(&self.excluded_proxy_endpoints);
|
||||
let provider = self
|
||||
.engine
|
||||
.providers()
|
||||
@@ -1230,9 +1239,23 @@ where
|
||||
}
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
Err(mut error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
if self.continuation_attempt == ContinuationAttempt::RotateForClientReplay
|
||||
&& error.continuation_recovery_disposition()
|
||||
== Some(ContinuationRecoveryDisposition::ClientReplayRequired)
|
||||
&& let Some(original) = self.last_retryable_failure.take()
|
||||
{
|
||||
// 迁移只改变客户端恢复指令;失败分类、发送事实与运维正文仍属于原请求
|
||||
error = original.with_client_recovery_from(error);
|
||||
self.last_retryable_failure_events.clear();
|
||||
}
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& self
|
||||
.last_retryable_failure
|
||||
.as_ref()
|
||||
.and_then(ProviderError::unreachable_proxy)
|
||||
.is_none()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
@@ -1264,7 +1287,7 @@ where
|
||||
)
|
||||
// 跨 Provider 候选推进必然换号(账号行按 Provider 隔离),
|
||||
// 预算耗尽后不再推进,交回容量类失败的原有终态语义。
|
||||
&& self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
|
||||
&& self.has_account_rotation_budget()
|
||||
&& self.advance_provider_candidate();
|
||||
let retryable = self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
@@ -1668,19 +1691,21 @@ where
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
// 只有尚未发送的逻辑请求进入新增策略;已有发送后安全恢复保持原有路由规则
|
||||
let connection_retry_requested = self.current_send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
// 服务器建议不可截短;超出请求冻结的上限后沿用安全换号或原错误出口
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& error.allows_connection_rotation();
|
||||
// 服务器建议不可截短
|
||||
// 超出上限,或上游拒绝继续重试当前账号时,跳过同账号等待
|
||||
// 较短的 Retry-After 也直接沿用安全换号或原错误出口
|
||||
let retry_after_exceeds_limit = error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after());
|
||||
let same_account_retry_blocked = error.same_account_retry_blocked();
|
||||
// Provider 的有界同账号传输恢复是尽力重试策略,不要求把 Sent/Ambiguous
|
||||
// 改写成安全证明;外部副作用、取消、预算与下游提交仍由 Core 拦截
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { transport })
|
||||
if !execution_effect_observed
|
||||
if !same_account_retry_blocked
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending
|
||||
@@ -1698,7 +1723,8 @@ where
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if !retry_after_exceeds_limit
|
||||
}) if !same_account_retry_blocked
|
||||
&& !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
@@ -1707,7 +1733,8 @@ where
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if !retry_after_exceeds_limit
|
||||
if !same_account_retry_blocked
|
||||
&& !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
@@ -1723,9 +1750,8 @@ where
|
||||
let can_rotate_connection = connection_retry_requested
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
|
||||
&& self.has_account_rotation_budget();
|
||||
// A 先恢复一次,再利用预留预算重选;下一次传输仍由 Provider 的恢复策略决定
|
||||
let connection_rotation = transport_recovery
|
||||
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
|
||||
let reserve_connection_rotation = can_rotate_connection
|
||||
@@ -1754,7 +1780,11 @@ where
|
||||
max_delay,
|
||||
jitter,
|
||||
..
|
||||
}) if ordinary_retry && !suppress_transient_retry && !retry_after_exceeds_limit => {
|
||||
}) if !same_account_retry_blocked
|
||||
&& ordinary_retry
|
||||
&& !suppress_transient_retry
|
||||
&& !retry_after_exceeds_limit =>
|
||||
{
|
||||
let retries = self
|
||||
.transient_retry_counts
|
||||
.entry(current.metadata.provider_account_id().clone())
|
||||
@@ -1776,7 +1806,8 @@ where
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let same_account_retry = !execution_effect_observed
|
||||
let same_account_retry = !same_account_retry_blocked
|
||||
&& !execution_effect_observed
|
||||
&& error.retries_same_account()
|
||||
&& provider_proved_replay_safe
|
||||
&& !self.replay_barrier_reached
|
||||
@@ -1797,8 +1828,7 @@ where
|
||||
&& !is_return_error_retry(&error)
|
||||
&& transport_recovery.is_none()
|
||||
&& (ordinary_retry || account_rotation_retry)
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
&& self.has_account_rotation_budget();
|
||||
let retryable = !error.retry_is_prohibited()
|
||||
&& (continuation_retry
|
||||
|| same_account_retry
|
||||
@@ -1834,6 +1864,7 @@ where
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"retryAfterExceeded": retry_after_exceeds_limit,
|
||||
"sameAccountRetryBlocked": same_account_retry_blocked,
|
||||
"maxRetryAfterSeconds": self.plan.max_retry_after().as_secs(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
@@ -1988,6 +2019,11 @@ where
|
||||
}
|
||||
StreamRetry::Pending(retry) => self.pending_retry = Some(retry),
|
||||
StreamRetry::ConnectionRotation(retry) => {
|
||||
if let Some(endpoint) = pending.error.unreachable_proxy()
|
||||
&& !self.excluded_proxy_endpoints.contains(endpoint)
|
||||
{
|
||||
self.excluded_proxy_endpoints.push(endpoint.clone());
|
||||
}
|
||||
self.excluded_accounts.insert(retry.account.clone());
|
||||
self.connection_rotation = Some(retry);
|
||||
}
|
||||
@@ -2174,6 +2210,26 @@ where
|
||||
}
|
||||
|
||||
match self.continuation_attempt {
|
||||
ContinuationAttempt::Native
|
||||
if error.continuation_recovery_disposition()
|
||||
== Some(ContinuationRecoveryDisposition::ClientReplayRequired)
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransientRetry {
|
||||
policy: crate::error::TransientRetryPolicy::RetryThenRotateAccount,
|
||||
..
|
||||
})
|
||||
)
|
||||
&& error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after())
|
||||
&& !error.retry_is_prohibited()
|
||||
&& self.has_account_rotation_budget() =>
|
||||
{
|
||||
self.continuation_attempt = ContinuationAttempt::RotateForClientReplay;
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
ContinuationAttempt::Native => match error.continuation_recovery_disposition() {
|
||||
Some(ContinuationRecoveryDisposition::RetryExactConnection) => {
|
||||
self.recovery_account = Some(current.metadata.provider_account_id().clone());
|
||||
@@ -2197,8 +2253,7 @@ where
|
||||
// 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放,
|
||||
// 落回不可重试路径以原始上游错误终态。
|
||||
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny
|
||||
if self.account_rotations
|
||||
>= self.plan.account_selection_policy().max_account_rotations() =>
|
||||
if !self.has_account_rotation_budget() =>
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -2211,7 +2266,7 @@ where
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
ContinuationAttempt::None => return false,
|
||||
ContinuationAttempt::None | ContinuationAttempt::RotateForClientReplay => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -2250,7 +2305,7 @@ where
|
||||
return false;
|
||||
}
|
||||
// Native 期间仍可能配置为禁止换号,跨账号重放共用请求冻结的预算
|
||||
if self.account_rotations >= self.plan.account_selection_policy().max_account_rotations() {
|
||||
if !self.has_account_rotation_budget() {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -801,16 +801,38 @@ impl DefaultExecutionService {
|
||||
active_request,
|
||||
admission_decision_ms,
|
||||
} = start_guard;
|
||||
let observation = plan
|
||||
.candidates()
|
||||
.first()
|
||||
.map_or_else(Default::default, |candidate| {
|
||||
providers.request_observation(
|
||||
candidate.provider(),
|
||||
&operation,
|
||||
client.policy.key_id(),
|
||||
)
|
||||
let observation_provider = match &target {
|
||||
ExecutionTarget::ProviderEndpoint { provider, .. } => Some(provider),
|
||||
ExecutionTarget::Model(_) => plan
|
||||
.candidates()
|
||||
.iter()
|
||||
.find(|candidate| {
|
||||
operation
|
||||
.provider_session_state(candidate.provider().as_str())
|
||||
.is_some_and(|state| state.extension_owner().is_none())
|
||||
})
|
||||
.or_else(|| plan.candidates().first())
|
||||
.map(|candidate| candidate.provider()),
|
||||
};
|
||||
let provider_observation =
|
||||
observation_provider.map_or_else(Default::default, |candidate| {
|
||||
providers.request_observation(candidate, &operation, client.policy.key_id())
|
||||
});
|
||||
// 纯协议事实不依赖候选顺序,Provider 仅补充所属会话的旧配置和恢复关联
|
||||
let observation = match &operation {
|
||||
Operation::Generate(request) => {
|
||||
let payload = request.protocol_payload();
|
||||
gateway_protocol::request::request_observation(
|
||||
payload.protocol(),
|
||||
payload.body(),
|
||||
payload.context(),
|
||||
provider_observation
|
||||
.previous_reasoning_effort_override
|
||||
.as_deref(),
|
||||
)
|
||||
}
|
||||
_ => Default::default(),
|
||||
};
|
||||
let (request_kind, subagent_kind) = if let Some(nested) = authorization.nested.as_ref() {
|
||||
(
|
||||
Some("plugin_child_model".to_owned()),
|
||||
@@ -843,7 +865,9 @@ impl DefaultExecutionService {
|
||||
operation: operation.kind(),
|
||||
endpoint: metadata.endpoint,
|
||||
client_transport: metadata.transport.as_str().to_owned(),
|
||||
requested_model: target.into_public_model().or(observation.requested_model),
|
||||
requested_model: target
|
||||
.into_public_model()
|
||||
.or(provider_observation.requested_model),
|
||||
client_ip: metadata.client_ip,
|
||||
user_agent: metadata.user_agent,
|
||||
reasoning_effort: observation.reasoning_effort,
|
||||
@@ -851,7 +875,7 @@ impl DefaultExecutionService {
|
||||
request_kind,
|
||||
subagent_kind,
|
||||
compact: observation.compact,
|
||||
continuation: observation.continuation,
|
||||
continuation: provider_observation.continuation,
|
||||
image_generation_requested: operation.image_generation_requested(),
|
||||
admission_decision_ms,
|
||||
started_at,
|
||||
|
||||
@@ -124,6 +124,7 @@ impl AttemptTrigger {
|
||||
/// 一旦设置,Provider 在本 attempt 内不得换号或切换 target
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccountAttemptContext {
|
||||
excluded_proxy_endpoints: Vec<crate::upstream::ProxyEndpoint>,
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: Option<ProviderAccountId>,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
@@ -140,6 +141,7 @@ impl AccountAttemptContext {
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account,
|
||||
state_owner,
|
||||
@@ -160,6 +162,7 @@ impl AccountAttemptContext {
|
||||
) -> Self {
|
||||
let (required_account, eligibility) = target.into_selection();
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account: Some(required_account),
|
||||
state_owner,
|
||||
@@ -265,6 +268,8 @@ pub enum ContinuationAttempt {
|
||||
///
|
||||
/// 该枚举本身不证明客户端 input 完整,也不授权 Provider 清除 previous-response
|
||||
ReplayAny,
|
||||
/// 仅迁移账号绑定,随后要求客户端携带完整历史重放;不得发送当前增量输入
|
||||
RotateForClientReplay,
|
||||
}
|
||||
|
||||
impl ContinuationAttempt {
|
||||
@@ -275,6 +280,7 @@ impl ContinuationAttempt {
|
||||
Self::Native => "native",
|
||||
Self::ReplayOwner => "replay_owner",
|
||||
Self::ReplayAny => "replay_any",
|
||||
Self::RotateForClientReplay => "rotate_for_client_replay",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -593,7 +599,9 @@ impl AttemptContext {
|
||||
.account_state_owner()
|
||||
.filter(|owner| owner.provider() == provider)
|
||||
.map(|owner| owner.account()),
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny => None,
|
||||
ContinuationAttempt::None
|
||||
| ContinuationAttempt::ReplayAny
|
||||
| ContinuationAttempt::RotateForClientReplay => None,
|
||||
});
|
||||
self.request.upstream_adapters.as_ref().map_or(
|
||||
Ok(upstream_adapter::UpstreamAdapterSelection::Native),
|
||||
@@ -896,6 +904,17 @@ impl AttemptContext {
|
||||
self.account.excluded_accounts()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn excludes_proxy(&self, proxy: Option<&crate::account::OutboundProxy>) -> bool {
|
||||
!self.account.excluded_proxy_endpoints.is_empty()
|
||||
&& proxy.is_some_and(|proxy| {
|
||||
self.account
|
||||
.excluded_proxy_endpoints
|
||||
.iter()
|
||||
.any(|endpoint| proxy.matches_endpoint(endpoint))
|
||||
})
|
||||
}
|
||||
|
||||
/// 管理端诊断或请求局部恢复 attempt 强制使用的唯一账号
|
||||
#[must_use]
|
||||
pub const fn required_account(&self) -> Option<&ProviderAccountId> {
|
||||
|
||||
@@ -1088,17 +1088,13 @@ pub struct ProviderRequest {
|
||||
candidate: ProviderCandidate,
|
||||
}
|
||||
|
||||
/// Provider 对公共观测表可解释的请求语义;未知字段保持空值
|
||||
/// Provider 私有状态与独立端点的观测补充,通用请求语义由协议层提取
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProviderRequestObservation {
|
||||
/// 独立 Provider 端点声明的请求模型,仅用于观测,不参与文本模型目录路由
|
||||
pub requested_model: Option<PublicModelId>,
|
||||
/// Provider 从客户端请求与续接配置中解析的推理强度
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: bool,
|
||||
/// 已核实归属的上一响应配置;当前请求是否继承由协议层判断
|
||||
pub previous_reasoning_effort_override: Option<String>,
|
||||
pub continuation: ContinuationRequestObservation,
|
||||
}
|
||||
|
||||
|
||||
@@ -117,7 +117,8 @@ pub enum ContinuationFailure {
|
||||
pub enum ContinuationRecoveryDisposition {
|
||||
/// 只能再次取得同账号、同一精确连接,不能改变 continuation scope
|
||||
RetryExactConnection,
|
||||
/// 代理停止内部恢复,由客户端携带完整输入创建新链
|
||||
/// 当前增量不能由代理重放,由客户端携带完整输入创建新链
|
||||
/// 显式换号策略仍可迁移会话绑定,但不能发送增量输入
|
||||
ClientReplayRequired,
|
||||
/// Provider 已证明可以进入 owner 或其他合法目标的恢复流程
|
||||
ProviderReplayAllowed,
|
||||
@@ -135,6 +136,7 @@ pub enum TransientRetryPolicy {
|
||||
}
|
||||
|
||||
/// Provider 在下游尚未提交事件时请求的隐藏恢复方式
|
||||
/// 传输恢复默认保持原账号;明确允许建连换号时,Core 可在同账号恢复失败后重选
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum PreDeliveryRetry {
|
||||
/// 排除本次账号,按普通调度策略重新选号
|
||||
@@ -152,7 +154,7 @@ pub enum PreDeliveryRetry {
|
||||
/// 同账号恢复策略,不提供额外安全重放资格
|
||||
policy: TransientRetryPolicy,
|
||||
},
|
||||
/// 固定本次账号,并按 Provider 给出的序号重试当前传输
|
||||
/// 按 Provider 给出的序号重试当前传输
|
||||
/// 可用于发送结果不明确时的有界尽力恢复,不等于上游未执行的证明
|
||||
SameAccountTransportRetry {
|
||||
/// Provider-owned 传输重试序号,从 1 开始
|
||||
@@ -165,7 +167,7 @@ pub enum PreDeliveryRetry {
|
||||
/// Provider 明确指定实际失败的传输,避免 HTTP 恢复重新进入 WS
|
||||
transport: crate::upstream::AttemptTransport,
|
||||
},
|
||||
/// 固定本次账号,并要求 Provider 使用备用传输
|
||||
/// 要求 Provider 使用备用传输
|
||||
SameAccountTransportFallback,
|
||||
}
|
||||
|
||||
@@ -516,6 +518,8 @@ pub struct ProviderError {
|
||||
failure_observation: Option<Box<ProviderErrorFailureObservation>>,
|
||||
replay_safe: bool,
|
||||
retry_prohibited: bool,
|
||||
same_account_retry_blocked: bool,
|
||||
connection_rotation_allowed: bool,
|
||||
pre_delivery_retry: Option<Box<PreDeliveryRetry>>,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
@@ -537,6 +541,7 @@ struct ProviderErrorUpstreamValues {
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorFailureObservation {
|
||||
unreachable_proxy: Option<crate::upstream::ProxyEndpoint>,
|
||||
continuation_unavailable_reason: Option<&'static str>,
|
||||
connection: Option<ProviderConnectionObservation>,
|
||||
}
|
||||
@@ -559,6 +564,8 @@ impl ProviderError {
|
||||
failure_observation: None,
|
||||
replay_safe: false,
|
||||
retry_prohibited: false,
|
||||
same_account_retry_blocked: false,
|
||||
connection_rotation_allowed: false,
|
||||
pre_delivery_retry: None,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
@@ -657,6 +664,20 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 仅当连接账号绑定代理的入口失败时记录;不用于持久化或输出地址
|
||||
#[must_use]
|
||||
pub fn with_unreachable_proxy(mut self, endpoint: crate::upstream::ProxyEndpoint) -> Self {
|
||||
self.failure_observation_mut().unreachable_proxy = Some(endpoint);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn unreachable_proxy(&self) -> Option<&crate::upstream::ProxyEndpoint> {
|
||||
self.failure_observation
|
||||
.as_deref()
|
||||
.and_then(|observation| observation.unreachable_proxy.as_ref())
|
||||
}
|
||||
|
||||
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
|
||||
self.failure_observation
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
|
||||
@@ -685,6 +706,22 @@ impl ProviderError {
|
||||
self.retry_prohibited
|
||||
}
|
||||
|
||||
/// 保留已选择的耗尽策略,但禁止执行同账号退避和传输恢复
|
||||
///
|
||||
/// Provider 在上游明确拒绝继续使用当前账号时设置
|
||||
/// Core 不根据错误码自行推断
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry_blocked(mut self) -> Self {
|
||||
self.same_account_retry_blocked = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 返回上游是否拒绝继续重试当前账号
|
||||
#[must_use]
|
||||
pub const fn same_account_retry_blocked(&self) -> bool {
|
||||
self.same_account_retry_blocked
|
||||
}
|
||||
|
||||
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复
|
||||
///
|
||||
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
|
||||
@@ -740,9 +777,21 @@ impl ProviderError {
|
||||
self.pre_delivery_retry = Some(Box::new(PreDeliveryRetry::SameAccountConnectionRetry {
|
||||
transport,
|
||||
}));
|
||||
self.allow_connection_rotation();
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 为瞬时建连失败预留换号机会,不授予重放资格或增加传输重试次数
|
||||
pub fn allow_connection_rotation(&mut self) {
|
||||
self.connection_rotation_allowed = true;
|
||||
}
|
||||
|
||||
/// 供 Core 在传输仍可恢复时为换号预留建连预算
|
||||
#[must_use]
|
||||
pub const fn allows_connection_rotation(&self) -> bool {
|
||||
self.connection_rotation_allowed
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -975,6 +1024,15 @@ impl ProviderError {
|
||||
self.client_visible_upstream_response.as_deref()
|
||||
}
|
||||
|
||||
/// 绑定迁移完成后仅替换客户端恢复投影,保留原始失败的诊断与计费事实
|
||||
pub(crate) fn with_client_recovery_from(mut self, recovery: Self) -> Self {
|
||||
self.client_visible_upstream_error = recovery.client_visible_upstream_error;
|
||||
self.client_visible_upstream_response = recovery.client_visible_upstream_response;
|
||||
self.continuation_failure = recovery.continuation_failure;
|
||||
self.continuation_recovery_disposition = recovery.continuation_recovery_disposition;
|
||||
self
|
||||
}
|
||||
|
||||
/// 取走只供本次重试/提交决策使用的原子客户端事件
|
||||
///
|
||||
/// Core 必须在创建稳定快照、记录中间失败或构造终态前调用本方法,避免原始
|
||||
@@ -1008,6 +1066,8 @@ impl ProviderError {
|
||||
failure_observation: self.failure_observation.clone(),
|
||||
replay_safe: self.replay_safe,
|
||||
retry_prohibited: self.retry_prohibited,
|
||||
same_account_retry_blocked: self.same_account_retry_blocked,
|
||||
connection_rotation_allowed: self.connection_rotation_allowed,
|
||||
pre_delivery_retry: self.pre_delivery_retry.clone(),
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
retry_same_account: self.retry_same_account,
|
||||
@@ -1036,6 +1096,10 @@ impl fmt::Debug for ProviderError {
|
||||
)
|
||||
.field("retry_after", &self.retry_after)
|
||||
.field("retry_prohibited", &self.retry_prohibited)
|
||||
.field(
|
||||
"same_account_retry_blocked",
|
||||
&self.same_account_retry_blocked,
|
||||
)
|
||||
.field("continuation_failure", &self.continuation_failure)
|
||||
.field(
|
||||
"continuation_recovery_disposition",
|
||||
@@ -1047,6 +1111,10 @@ impl fmt::Debug for ProviderError {
|
||||
)
|
||||
.field("connection_observation", &self.connection_observation())
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field(
|
||||
"connection_rotation_allowed",
|
||||
&self.connection_rotation_allowed,
|
||||
)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
"credential_recovery_required",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! 这里只保留网关需要解释、路由或结算的稳定语义
|
||||
//! 协议 adapter 无法写入
|
||||
//! wire body 与连接级事实随 [`ProtocolPayload`] 或 [`RawJsonPayload`] 不透明传递,
|
||||
//! 由对应 Provider 自己解释
|
||||
//! 纯协议事实由协议层解析,私有字段与会话由对应 Provider 解释
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
@@ -458,8 +458,8 @@ impl ClientTurn {
|
||||
impl GenerateRequest {
|
||||
/// 创建携带协议不透明正文的生成请求
|
||||
///
|
||||
/// 完整请求只以原始 wire object 保存;Core 仅局部只读已知字段以推导路由能力,
|
||||
/// Provider 负责解释与转换正文
|
||||
/// 完整请求以原始 wire object 保存,协议层提取路由与权限所需的纯事实
|
||||
/// Provider 在选定账号后编排协议转换与私有字段处理
|
||||
#[must_use]
|
||||
pub fn from_protocol_payload(protocol_payload: ProtocolPayload) -> Self {
|
||||
Self {
|
||||
@@ -521,9 +521,7 @@ impl GenerateRequest {
|
||||
/// 返回客户端是否请求了图片生成工具
|
||||
#[must_use]
|
||||
pub fn image_generation_requested(&self) -> bool {
|
||||
self.payload.source_image_generation
|
||||
|| (self.protocol_payload().protocol() == "openai"
|
||||
&& requests_image_generation(self.body()))
|
||||
self.payload.source_image_generation || self.payload.wire_facts.image_generation
|
||||
}
|
||||
|
||||
/// 返回指定 Provider 的连接内会话状态
|
||||
@@ -596,35 +594,6 @@ impl fmt::Debug for GenerateRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 原生工具声明授权上游生成图片,Codex 的被动 namespace 与图片输入不属于该权限
|
||||
fn requests_image_generation(body: &Map<String, Value>) -> bool {
|
||||
contains_image_tool(body.get("tools"))
|
||||
|| body
|
||||
.get("input")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|items| {
|
||||
items.iter().any(|item| {
|
||||
item.get("type").and_then(Value::as_str) == Some("additional_tools")
|
||||
&& contains_image_tool(item.get("tools"))
|
||||
})
|
||||
})
|
||||
|| body.get("tool_choice").is_some_and(|choice| {
|
||||
is_image_tool(choice)
|
||||
|| (choice.get("type").and_then(Value::as_str) == Some("allowed_tools")
|
||||
&& contains_image_tool(choice.get("tools")))
|
||||
})
|
||||
}
|
||||
|
||||
fn contains_image_tool(value: Option<&Value>) -> bool {
|
||||
value
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|tools| tools.iter().any(is_image_tool))
|
||||
}
|
||||
|
||||
fn is_image_tool(value: &Value) -> bool {
|
||||
value.get("type").and_then(Value::as_str) == Some("image_generation")
|
||||
}
|
||||
|
||||
/// 图像 API 的稳定端点语义
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ImageRequestKind {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Provider 会话亲和、绑定和排除事实
|
||||
|
||||
use super::{ProviderStoreError, ProviderStoreErrorKind};
|
||||
use crate::{account::ProviderAccountId, identity::ProviderKind};
|
||||
use crate::{account::ProviderAccountId, identity::ProviderKind, policy::ClientApiKeyId};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{collections::BTreeSet, fmt, time::Duration};
|
||||
|
||||
@@ -81,6 +81,36 @@ pub struct ProviderSessionAlias {
|
||||
pub follow_only: bool,
|
||||
}
|
||||
|
||||
/// 发送前需要核对版本的会话绑定;首次认领的 expected 为空
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ProviderSessionClaim<'a> {
|
||||
pub key: &'a ProviderSessionAffinityKey,
|
||||
pub expected: Option<&'a ProviderSessionBinding>,
|
||||
pub account: &'a ProviderAccountId,
|
||||
pub ttl: Duration,
|
||||
}
|
||||
|
||||
/// 仅成功的内置轮询推进游标;有会话的分配必须同时提交绑定
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum ProviderAccountAdmission<'a> {
|
||||
Session(ProviderSessionClaim<'a>),
|
||||
RoundRobin {
|
||||
client_key: &'a ClientApiKeyId,
|
||||
cursor: u64,
|
||||
session: Option<ProviderSessionClaim<'a>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl<'a> ProviderAccountAdmission<'a> {
|
||||
#[must_use]
|
||||
pub const fn session(self) -> Option<ProviderSessionClaim<'a>> {
|
||||
match self {
|
||||
Self::Session(session) => Some(session),
|
||||
Self::RoundRobin { session, .. } => session,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 客户端作用域内的会话账号绑定,原始会话身份由 Provider 哈希后传入
|
||||
pub trait ProviderSessionAffinityPort: Send + Sync {
|
||||
fn load<'a>(
|
||||
@@ -89,16 +119,19 @@ pub trait ProviderSessionAffinityPort: Send + Sync {
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionBinding>, ProviderStoreError>>;
|
||||
|
||||
/// 在发送前原子认领、续期或迁移;None 表示快照已变化,调用方必须释放租约重新选择
|
||||
/// expected 为 None 只允许首次认领,已有快照必须连同版本匹配
|
||||
fn compare_and_bind<'a>(
|
||||
/// 读取本 Client Key 和 Provider 的下一轮位置,不推进游标
|
||||
fn round_robin_cursor<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
expected: Option<&'a ProviderSessionBinding>,
|
||||
account_id: &'a ProviderAccountId,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionBinding>, ProviderStoreError>>;
|
||||
provider: &'a ProviderKind,
|
||||
client_key: &'a ClientApiKeyId,
|
||||
) -> BoxFuture<'a, Result<u64, ProviderStoreError>>;
|
||||
|
||||
/// 原子核对并提交会话版本和轮询位置;冲突时均不写入,调用方释放租约重选
|
||||
fn admit_account<'a>(
|
||||
&'a self,
|
||||
provider: &'a ProviderKind,
|
||||
admission: ProviderAccountAdmission<'a>,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 显式观测到的请求关联只指向会话键,不缓存账号,迁移后仍读取当前绑定
|
||||
fn load_alias<'a>(
|
||||
|
||||
@@ -154,10 +154,16 @@ impl ProviderSchedulingLeaseRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lease 生命周期由具体 Store guard 管理,Provider 只能持有
|
||||
pub trait ProviderLeaseGuard: Send + Sync + 'static {}
|
||||
/// Drop 回收租约;同一请求需要重选时显式等待释放,避免被自己的租约阻塞
|
||||
pub trait ProviderLeaseGuard: Send + Sync + 'static {
|
||||
fn release(self: Box<Self>) -> BoxFuture<'static, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
impl<T> ProviderLeaseGuard for T where T: Send + Sync + 'static {}
|
||||
impl ProviderLeaseGuard for () {
|
||||
fn release(self: Box<Self>) -> BoxFuture<'static, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
pub enum ProviderLeaseAcquisition {
|
||||
Acquired(Box<dyn ProviderLeaseGuard>),
|
||||
|
||||
@@ -10,8 +10,9 @@ pub use lease::{
|
||||
};
|
||||
mod affinity;
|
||||
pub use affinity::{
|
||||
ProviderSessionAffinityKey, ProviderSessionAffinityPort, ProviderSessionAlias,
|
||||
ProviderSessionBinding, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
ProviderAccountAdmission, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
ProviderSessionAlias, ProviderSessionBinding, ProviderSessionClaim,
|
||||
ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
};
|
||||
mod catalog;
|
||||
pub use catalog::{
|
||||
|
||||
@@ -5,6 +5,14 @@ use crate::{account::OpaqueProviderData, identity::ProviderKind, routing::Config
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
/// 后台模型操作读取 Provider 全局画像,不读取单个 Client Key 的覆盖
|
||||
fn load_request_profile<'a>(
|
||||
&'a self,
|
||||
_provider: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<Option<OpaqueProviderData>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
/// 原子推进预热执行游标,重启或时钟回拨后不重复领取已消费的时刻
|
||||
fn claim_warmup_slot<'a>(
|
||||
&'a self,
|
||||
@@ -52,7 +60,7 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验五段 Cron 预热计划
|
||||
/// 校验单条五段 Cron 表达式
|
||||
#[must_use]
|
||||
pub fn valid_warmup_cron(value: &str) -> bool {
|
||||
parse_warmup_cron(value).is_some()
|
||||
@@ -73,7 +81,7 @@ fn parse_warmup_cron(value: &str) -> Option<cron::Schedule> {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderWarmupPolicy {
|
||||
enabled: bool,
|
||||
cron: String,
|
||||
schedules: Vec<cron::Schedule>,
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
@@ -84,29 +92,47 @@ impl ProviderWarmupPolicy {
|
||||
cron: &str,
|
||||
model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
if (enabled || !cron.is_empty()) && !valid_warmup_cron(cron) {
|
||||
Self::validate_and_parse(enabled, cron, model).map(|_| ())
|
||||
}
|
||||
|
||||
fn validate_and_parse(
|
||||
enabled: bool,
|
||||
cron: &str,
|
||||
model: Option<&str>,
|
||||
) -> Result<Vec<cron::Schedule>, &'static str> {
|
||||
if cron.len() > 255 || (enabled && cron.is_empty()) {
|
||||
return Err("account_warmup_cron");
|
||||
}
|
||||
// 分号分隔独立表达式,逗号保留给 Cron 字段内的列表
|
||||
let schedules = if cron.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
cron.split(';')
|
||||
.map(parse_warmup_cron)
|
||||
.collect::<Option<Vec<_>>>()
|
||||
.ok_or("account_warmup_cron")?
|
||||
};
|
||||
if (enabled && model.is_none()) || !valid_optional_probe_model(model) {
|
||||
return Err("account_warmup_model");
|
||||
}
|
||||
Ok(())
|
||||
Ok(schedules)
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
cron: String,
|
||||
cron: &str,
|
||||
model: Option<String>,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if Self::validate_values(enabled, &cron, model.as_deref()).is_err() {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate warmup policy",
|
||||
));
|
||||
}
|
||||
let schedules =
|
||||
Self::validate_and_parse(enabled, cron, model.as_deref()).map_err(|_| {
|
||||
ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate warmup policy",
|
||||
)
|
||||
})?;
|
||||
Ok(Self {
|
||||
enabled,
|
||||
cron,
|
||||
schedules,
|
||||
model,
|
||||
})
|
||||
}
|
||||
@@ -115,7 +141,7 @@ impl ProviderWarmupPolicy {
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
cron: String::new(),
|
||||
schedules: Vec::new(),
|
||||
model: None,
|
||||
}
|
||||
}
|
||||
@@ -125,11 +151,6 @@ impl ProviderWarmupPolicy {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn cron(&self) -> &str {
|
||||
&self.cron
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn model(&self) -> Option<&str> {
|
||||
self.model.as_deref()
|
||||
@@ -138,7 +159,10 @@ impl ProviderWarmupPolicy {
|
||||
/// 匹配部署时区中的整分钟时刻,时区重叠及执行去重由 Worker 与存储端口负责
|
||||
#[must_use]
|
||||
pub fn matches_slot(&self, slot: chrono::NaiveDateTime) -> bool {
|
||||
parse_warmup_cron(&self.cron).is_some_and(|schedule| schedule.includes(slot.and_utc()))
|
||||
let instant = slot.and_utc();
|
||||
self.schedules
|
||||
.iter()
|
||||
.any(|schedule| schedule.includes(instant))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1247,13 +1247,13 @@ impl RuntimeSnapshot {
|
||||
operation: &Operation,
|
||||
context: &RoutingContext,
|
||||
) -> Result<RoutingPlan, RoutingError> {
|
||||
self.plan_inner(
|
||||
public_model,
|
||||
operation,
|
||||
self.all_account_scope(),
|
||||
context,
|
||||
false,
|
||||
)
|
||||
// 探测没有 Client Key,沿用同一发布快照的全局画像
|
||||
let scope = self
|
||||
.all_account_scope()
|
||||
.as_ref()
|
||||
.clone()
|
||||
.with_request_profiles(self.settings().request_profiles().clone());
|
||||
self.plan_inner(public_model, operation, Arc::new(scope), context, false)
|
||||
}
|
||||
|
||||
fn plan_inner(
|
||||
|
||||
@@ -101,6 +101,7 @@ impl CompiledSettings {
|
||||
u64::from(settings.openai_session_affinity_ttl_hours) * 3600,
|
||||
))
|
||||
.with_smart_scheduling(settings.smart_scheduling)
|
||||
.with_quota_reset_window_weights(settings.quota_reset_window_weights)
|
||||
.with_queue(ConcurrencyQueuePolicy {
|
||||
max_waiting: settings.max_waiting_per_account,
|
||||
timeout: Duration::from_secs(u64::from(settings.account_queue_timeout_seconds)),
|
||||
|
||||
@@ -30,6 +30,8 @@ pub struct SettingsValues {
|
||||
pub(crate) openai_session_affinity_ttl_hours: u32,
|
||||
pub(crate) responses_max_decompressed_body_bytes: u64,
|
||||
pub(crate) smart_scheduling: crate::account::SmartSchedulingConfig,
|
||||
#[serde(default)]
|
||||
pub(crate) quota_reset_window_weights: crate::account::ResetWindowWeights,
|
||||
pub(crate) rotation_strategy: String,
|
||||
pub(crate) model_mappings: Arc<BTreeMap<String, String>>,
|
||||
pub(crate) min_codex_desktop_version: Option<String>,
|
||||
@@ -111,6 +113,15 @@ impl SettingsValues {
|
||||
&self.request_profiles
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_quota_reset_window_weights(
|
||||
mut self,
|
||||
weights: crate::account::ResetWindowWeights,
|
||||
) -> Self {
|
||||
self.quota_reset_window_weights = weights;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_smart_scheduling(
|
||||
mut self,
|
||||
@@ -201,6 +212,7 @@ impl SettingsValues {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: crate::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: crate::account::ResetWindowWeights::default(),
|
||||
rotation_strategy: rotation_strategy.into(),
|
||||
model_mappings: Arc::new(model_mappings),
|
||||
min_codex_desktop_version,
|
||||
|
||||
@@ -4,6 +4,32 @@ use std::{fmt, num::NonZeroU32};
|
||||
|
||||
use crate::validation::{IdentifierError, validate_text};
|
||||
|
||||
/// 拨号目标只按主机与端口识别;认证、协议与连接上限不改变 TCP 入口
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProxyEndpoint {
|
||||
host: String,
|
||||
port: u16,
|
||||
}
|
||||
|
||||
impl ProxyEndpoint {
|
||||
pub(crate) fn new(host: &str, port: u16) -> Self {
|
||||
Self {
|
||||
host: host.to_ascii_lowercase(),
|
||||
port,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, host: &str, port: u16) -> bool {
|
||||
self.port == port && host.eq_ignore_ascii_case(&self.host)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProxyEndpoint {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ProxyEndpoint(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// 上游是否可能已经收到业务 payload
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum UpstreamSendState {
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
mod location;
|
||||
mod model_access;
|
||||
mod proxy;
|
||||
mod quota_reset_priority;
|
||||
mod quota_settings;
|
||||
mod scope;
|
||||
mod selection;
|
||||
mod selection_diagnostics;
|
||||
@@ -56,6 +58,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
in_flight,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: remaining,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
@@ -228,6 +231,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
@@ -247,6 +251,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
@@ -641,6 +646,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
in_flight: 2,
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
|
||||
@@ -15,6 +15,35 @@ fn proxy_endpoints_support_explicit_schemes_and_redact_credentials() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dial_endpoints_match_host_and_port_without_credentials_or_capacity() {
|
||||
for (source, same, different) in [
|
||||
(
|
||||
"http://user:password@Proxy.Example",
|
||||
"socks5://other:secret@proxy.example:80",
|
||||
"http://proxy.example:81",
|
||||
),
|
||||
(
|
||||
"https://user:password@[::1]",
|
||||
"http://other:secret@[::1]:443",
|
||||
"https://[::2]",
|
||||
),
|
||||
] {
|
||||
let endpoint = OutboundProxy::parse(source).unwrap().dial_endpoint();
|
||||
let same = OutboundProxy::parse(same)
|
||||
.unwrap()
|
||||
.with_max_websocket_connections(3);
|
||||
assert_eq!(endpoint, same.dial_endpoint());
|
||||
assert!(same.matches_endpoint(&endpoint));
|
||||
assert!(
|
||||
!OutboundProxy::parse(different)
|
||||
.unwrap()
|
||||
.matches_endpoint(&endpoint)
|
||||
);
|
||||
assert_eq!(format!("{endpoint:?}"), "ProxyEndpoint(<redacted>)");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_proxy_never_silently_becomes_direct() {
|
||||
for url in [
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//! 验证重置窗口独立排序、相对系数及亲和与可用性边界
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::account::{
|
||||
AccountQuotaResetWindow, AccountSelector, QuotaResetPeriod, ResetWindowWeights,
|
||||
RotationStrategy,
|
||||
};
|
||||
|
||||
use super::{candidate, context};
|
||||
|
||||
#[test]
|
||||
fn selected_windows_rank_strictly_without_mixing_runtime_scores_or_rotating_near_ties() {
|
||||
let mut context = context(RotationStrategy::QuotaResetPriority);
|
||||
let mut early_weekly = candidate("acct_early_weekly", 2, Some(100));
|
||||
early_weekly.signals.failure_rate_basis_points = Some(9_000);
|
||||
early_weekly.signals.first_output_latency_ms = Some(20_000);
|
||||
let mut late_weekly = candidate("acct_late_weekly", 0, Some(9_900));
|
||||
late_weekly.signals.failure_rate_basis_points = Some(0);
|
||||
late_weekly.signals.first_output_latency_ms = Some(10);
|
||||
for (candidate, short, weekly) in [
|
||||
(&mut early_weekly, 14_400, 6 * 86_400),
|
||||
(&mut late_weekly, 3_600, 6 * 86_400 + 43_200),
|
||||
] {
|
||||
candidate.signals.quota_reset_at = Some(context.now + Duration::from_secs(short));
|
||||
candidate.signals.quota_reset_windows = Some(
|
||||
[
|
||||
(QuotaResetPeriod::FiveHour, short),
|
||||
(QuotaResetPeriod::Weekly, weekly),
|
||||
]
|
||||
.map(|(period, seconds)| AccountQuotaResetWindow {
|
||||
period,
|
||||
reset_at: context.now + Duration::from_secs(seconds),
|
||||
})
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let mut candidates = [early_weekly, late_weekly];
|
||||
for (weights, expected) in [
|
||||
([0.0, 0.0, 0.0], "acct_late_weekly"),
|
||||
([1.0, 0.0, 0.0], "acct_late_weekly"),
|
||||
([0.0, 0.1, 0.0], "acct_early_weekly"),
|
||||
([0.0, 1.0, 0.0], "acct_early_weekly"),
|
||||
([0.0, 10.0, 0.0], "acct_early_weekly"),
|
||||
([1.0, 0.1, 0.0], "acct_late_weekly"),
|
||||
([10.0, 1.0, 0.0], "acct_late_weekly"),
|
||||
([0.1, 10.0, 0.0], "acct_early_weekly"),
|
||||
] {
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_quota_reset_window_weights(ResetWindowWeights::new(weights).unwrap());
|
||||
for cursor in 0..6 {
|
||||
context.round_robin_cursor = cursor;
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
expected,
|
||||
"weights {weights:?}, cursor {cursor}"
|
||||
);
|
||||
}
|
||||
}
|
||||
context.preferred_account = Some(candidates[1].account.id().clone());
|
||||
let selected = AccountSelector.select(&candidates, &context).unwrap();
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_late_weekly"
|
||||
);
|
||||
assert_eq!(
|
||||
selected.preferred(),
|
||||
gateway_core::account::PreferredAccountSelection::Hit
|
||||
);
|
||||
|
||||
context.preferred_account = None;
|
||||
candidates[0].signals.in_flight = 3;
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
"acct_late_weekly"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_elapsed_selected_windows_do_not_fall_back_to_disabled_periods() {
|
||||
let mut context = context(RotationStrategy::QuotaResetPriority);
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_quota_reset_window_weights(ResetWindowWeights::new([0.0, 0.0, 1.0]).unwrap());
|
||||
let mut known = candidate("acct_known_monthly", 1, None);
|
||||
known.signals.quota_reset_windows = Some(
|
||||
vec![AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Monthly,
|
||||
reset_at: context.now + Duration::from_secs(30 * 86_400),
|
||||
}]
|
||||
.into(),
|
||||
);
|
||||
for reset_at in [
|
||||
None,
|
||||
Some(context.now),
|
||||
Some(context.now - Duration::from_secs(1)),
|
||||
] {
|
||||
let mut missing = candidate("acct_earlier_other_period", 0, None);
|
||||
missing.signals.quota_reset_at = Some(context.now + Duration::from_secs(1));
|
||||
missing.signals.quota_reset_windows = Some(
|
||||
reset_at
|
||||
.into_iter()
|
||||
.map(|reset_at| AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Monthly,
|
||||
reset_at,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.into(),
|
||||
);
|
||||
let candidates = [missing, known.clone()];
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
"acct_known_monthly"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//! 额度桶配置校验与上限、预热的独立行为
|
||||
|
||||
use gateway_core::account::AccountQuotaSettings;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn warmup_and_limit_can_be_configured_independently() {
|
||||
let settings: AccountQuotaSettings = serde_json::from_value(json!({
|
||||
"codex:18000s": {"warmup": {"model": "test-model"}},
|
||||
"codex:604800s": {"limitPercent": 95}
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(settings.has_warmup());
|
||||
assert!(!settings.reached("codex:18000s", Some(100.0)));
|
||||
assert!(!settings.reached("codex:604800s", Some(94.9)));
|
||||
assert!(settings.reached("codex:604800s", Some(95.0)));
|
||||
assert!(!settings.reached("codex:604800s", None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_bucket_settings_are_rejected() {
|
||||
for invalid in [
|
||||
json!(95),
|
||||
json!({"limitPercent": 0}),
|
||||
json!({"limitPercent": 101}),
|
||||
json!({"warmup": {}}),
|
||||
json!({"warmup": {"model": ""}}),
|
||||
json!({"warmup": {"model": " model"}}),
|
||||
json!({"warmup": {"model": "model\nname"}}),
|
||||
json!({"warmup": {"model": "x".repeat(129)}}),
|
||||
json!({"warmup": {"model": "model", "unexpected": true}}),
|
||||
] {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountQuotaSettings>(json!({"codex:18000s": invalid}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
//! 验证智能调度权重、软亲和与容量约束的组合行为
|
||||
|
||||
use gateway_core::account::{
|
||||
AccountSelector, PreferredAccountSelection, RotationStrategy, SmartSchedulingConfig,
|
||||
AccountQuotaResetWindow, AccountSelector, PreferredAccountSelection, QuotaResetPeriod,
|
||||
RotationStrategy, SmartSchedulingConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -62,9 +63,10 @@ fn smart_config_rejects_invalid_weights_and_incomplete_wire_values() {
|
||||
#[test]
|
||||
fn reset_weight_rewards_only_future_resets_and_preserves_capacity_checks() {
|
||||
let mut context = context(RotationStrategy::Smart);
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_smart_scheduling(SmartSchedulingConfig::new([0.0, 0.0, 0.0, 0.0, 1.0, 0.0]).unwrap());
|
||||
context.policy = context.policy.with_smart_scheduling(serde_json::from_value(json!({
|
||||
"loadWeight":0, "quotaWeight":0, "healthWeight":0, "latencyWeight":0, "resetWeight":1, "queueWeight":0,
|
||||
"resetWindowWeights":{"fiveHour":0,"weekly":1,"monthly":0}
|
||||
})).unwrap());
|
||||
let mut candidates = [
|
||||
candidate_with_concurrency("acct_unknown", 0, 2),
|
||||
candidate_with_concurrency("acct_past", 0, 2),
|
||||
@@ -72,10 +74,20 @@ fn reset_weight_rewards_only_future_resets_and_preserves_capacity_checks() {
|
||||
candidate_with_concurrency("acct_near", 0, 2),
|
||||
candidate_with_concurrency("acct_far", 0, 2),
|
||||
];
|
||||
candidates[1].signals.quota_reset_at = Some(context.now - Duration::from_secs(1));
|
||||
candidates[2].signals.quota_reset_at = Some(context.now);
|
||||
candidates[3].signals.quota_reset_at = Some(context.now + Duration::from_secs(3600));
|
||||
candidates[4].signals.quota_reset_at = Some(context.now + Duration::from_secs(86_400));
|
||||
for (candidate, reset_at) in candidates[1..].iter_mut().zip([
|
||||
context.now - Duration::from_secs(1),
|
||||
context.now,
|
||||
context.now + Duration::from_secs(3600),
|
||||
context.now + Duration::from_secs(86_400),
|
||||
]) {
|
||||
candidate.signals.quota_reset_windows = Some(
|
||||
[AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Weekly,
|
||||
reset_at,
|
||||
}]
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
@@ -92,8 +104,8 @@ fn reset_weight_rewards_only_future_resets_and_preserves_capacity_checks() {
|
||||
let scores = snapshot["events"][0]["data"]["candidates"]
|
||||
.as_array()
|
||||
.unwrap();
|
||||
for (candidate, expected) in scores.iter().zip([0.0, 0.0, 0.0, 0.5, 0.04]) {
|
||||
assert_eq!(candidate["smartScore"].as_f64().unwrap(), expected);
|
||||
for (candidate, expected) in scores.iter().zip([0.0, 0.0, 0.0, 0.96, 0.5]) {
|
||||
assert!((candidate["smartScore"].as_f64().unwrap() - expected).abs() < 1e-10);
|
||||
}
|
||||
candidates[3].signals.in_flight = 2;
|
||||
assert_eq!(
|
||||
@@ -120,6 +132,36 @@ fn reset_weight_rewards_only_future_resets_and_preserves_capacity_checks() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_or_absent_window_weights_disable_reset_scores_and_tolerance() {
|
||||
let mut context = context(RotationStrategy::Smart);
|
||||
let mut candidate = candidate_with_concurrency("acct_reset", 0, 2);
|
||||
candidate.signals.quota_reset_windows = Some(
|
||||
[AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Monthly,
|
||||
reset_at: context.now + Duration::from_secs(1),
|
||||
}]
|
||||
.into(),
|
||||
);
|
||||
let legacy = json!({"loadWeight":0,"quotaWeight":0,"healthWeight":0,"latencyWeight":0,"resetWeight":10,"queueWeight":0});
|
||||
let mut disabled = legacy.clone();
|
||||
disabled["resetWindowWeights"] = json!({"fiveHour":0,"weekly":0,"monthly":0});
|
||||
for wire in [legacy, disabled] {
|
||||
let config: SmartSchedulingConfig = serde_json::from_value(wire).unwrap();
|
||||
context.policy = context.policy.with_smart_scheduling(config);
|
||||
let trace = gateway_core::diagnostics::TraceContext::new("req_zero_windows");
|
||||
trace.account_selection(std::slice::from_ref(&candidate), &context, None);
|
||||
let events = trace.snapshot().unwrap();
|
||||
let data = &events["events"][0]["data"];
|
||||
assert_eq!(data["candidates"][0]["smartScore"], 0.0);
|
||||
assert_eq!(data["smartScoreTolerance"], 0.0);
|
||||
assert_eq!(
|
||||
serde_json::to_value(config).unwrap()["resetWindowWeights"],
|
||||
json!({"fiveHour":0.0,"weekly":0.0,"monthly":0.0})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_weight_balances_live_pressure_and_health_only_when_enabled_for_smart() {
|
||||
for (strategy, queue_weight, expects_shortest) in [
|
||||
@@ -349,3 +391,41 @@ fn higher_weight_recovery_preserves_usable_soft_affinity() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_expired_and_duplicate_periods_do_not_inflate_reset_score() {
|
||||
let mut context = context(RotationStrategy::Smart);
|
||||
let mut candidate = candidate_with_concurrency("acct_partial", 0, 2);
|
||||
candidate.signals.quota_reset_at = Some(context.now + Duration::from_secs(1));
|
||||
candidate.signals.quota_reset_windows = Some(
|
||||
[
|
||||
(QuotaResetPeriod::Weekly, 86400),
|
||||
(QuotaResetPeriod::Weekly, 90000),
|
||||
(QuotaResetPeriod::Weekly, 0),
|
||||
(QuotaResetPeriod::FiveHour, 0),
|
||||
]
|
||||
.map(|(period, delta)| AccountQuotaResetWindow {
|
||||
period,
|
||||
reset_at: context.now + Duration::from_secs(delta),
|
||||
})
|
||||
.into(),
|
||||
);
|
||||
for scale in [1.0, 0.5] {
|
||||
let mut config = serde_json::to_value(
|
||||
SmartSchedulingConfig::new([0.0, 0.0, 0.0, 0.0, 1.0, 0.0]).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
config["resetWindowWeights"] = json!({
|
||||
"weekly": 2.0*scale, "fiveHour": 4.0*scale, "monthly": 4.0*scale
|
||||
});
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_smart_scheduling(serde_json::from_value(config).unwrap());
|
||||
let trace = gateway_core::diagnostics::TraceContext::new("req_partial");
|
||||
trace.account_selection(std::slice::from_ref(&candidate), &context, None);
|
||||
assert_eq!(
|
||||
trace.snapshot().unwrap()["events"][0]["data"]["candidates"][0]["smartScore"],
|
||||
0.1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1228,6 +1228,7 @@ fn plan_with_protection(
|
||||
None,
|
||||
)
|
||||
.with_smart_scheduling(account_selection_policy.smart_scheduling())
|
||||
.with_quota_reset_window_weights(account_selection_policy.quota_reset_window_weights())
|
||||
.with_max_account_rotations(account_selection_policy.max_account_rotations())
|
||||
.with_openai_account_affinity(account_selection_policy.openai_account_affinity())
|
||||
.with_openai_session_affinity_ttl_hours(
|
||||
@@ -2490,6 +2491,124 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
for (seconds, replacement, rotations, expected_attempts) in [
|
||||
(309, true, 1, 2),
|
||||
(309, false, 1, 2),
|
||||
(240, true, 1, 1),
|
||||
(309, true, 0, 1),
|
||||
] {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
unreachable!()
|
||||
};
|
||||
let operation =
|
||||
Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).unwrap(),
|
||||
));
|
||||
let route_plan = plan_with_protection(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::MIN)
|
||||
.with_max_account_rotations(rotations),
|
||||
Default::default(),
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
240,
|
||||
);
|
||||
let original_body = Bytes::from_static(br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"309","retry-after-ms":"309000"}}}"#);
|
||||
let original = transient_rejection()
|
||||
.with_retry_after(Duration::from_secs(seconds))
|
||||
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
|
||||
String::from_utf8(original_body.to_vec()).unwrap(),
|
||||
))
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
);
|
||||
let recovery = if replacement {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Resend full history.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
))
|
||||
} else {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(original)],
|
||||
},
|
||||
Script::Error(recovery),
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("client-previous"),
|
||||
PreviousResponseId::new("upstream-previous"),
|
||||
ClientApiKeyId::new("key_client_1").unwrap(),
|
||||
ProviderKind::new("openai").unwrap(),
|
||||
ProviderAccountId::new("acct_one").unwrap(),
|
||||
)
|
||||
.with_scope(gateway_core::engine::continuation::NativeContinuationScope::ConnectionLocal);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) =
|
||||
block_on(async { futures::poll!(&mut collection) })
|
||||
else {
|
||||
panic!("native capacity recovery must not wait on the old account");
|
||||
};
|
||||
drop(collection);
|
||||
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(seconds)));
|
||||
assert_eq!(
|
||||
error.raw_upstream_error().unwrap().as_str().as_bytes(),
|
||||
original_body.as_ref()
|
||||
);
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_error()
|
||||
.and_then(|detail| detail.code()),
|
||||
(replacement && expected_attempts == 2).then_some("previous_response_not_found")
|
||||
);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), expected_attempts);
|
||||
if contexts.len() == 2 {
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::RotateForClientReplay
|
||||
);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_one").unwrap())
|
||||
);
|
||||
assert!(contexts[1].required_account().is_none());
|
||||
}
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_client_replay_required_is_terminal() {
|
||||
for (kind, send_state) in [
|
||||
@@ -4325,6 +4444,87 @@ fn retry_after_above_the_configured_limit_rotates_without_waiting() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_more_retry_skips_same_account_wait_even_when_retry_after_is_short() {
|
||||
let advised = Duration::from_secs(30);
|
||||
for (rejection, rotates) in [
|
||||
(
|
||||
transient_rejection()
|
||||
.with_same_account_retry_blocked()
|
||||
.with_retry_after(advised),
|
||||
true,
|
||||
),
|
||||
(
|
||||
transient_rejection()
|
||||
.with_transient_retry(
|
||||
NonZeroU32::MIN,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
)
|
||||
.with_same_account_retry_blocked()
|
||||
.with_retry_after(advised),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, 240);
|
||||
let mut scripts = vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection)],
|
||||
}];
|
||||
if rotates {
|
||||
scripts.push(Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
});
|
||||
}
|
||||
let (coordinator, _, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let trace = session.trace().clone();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let polled = block_on(async { futures::poll!(&mut collection) });
|
||||
let trace = trace.snapshot().unwrap();
|
||||
let retry = trace["events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "retry.decided")
|
||||
.unwrap();
|
||||
assert_eq!(retry["data"]["sameAccountRetryBlocked"], true);
|
||||
assert_eq!(retry["data"]["transientRetry"], false);
|
||||
assert_eq!(retry["data"]["retryAfterExceeded"], false);
|
||||
if rotates {
|
||||
assert!(matches!(polled, std::task::Poll::Ready(Ok(_))));
|
||||
drop(collection);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(contexts[1].required_account(), None);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(
|
||||
polled,
|
||||
std::task::Poll::Ready(Err(EngineError::Provider(error)))
|
||||
if error.retry_after() == Some(advised)
|
||||
));
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excessive_retry_after_preserves_the_error_when_rotation_is_unavailable_or_disallowed() {
|
||||
for (rejection, rotation_allowed) in [
|
||||
|
||||
@@ -148,6 +148,209 @@ fn request(service: &DefaultExecutionService, transport: ClientTransport) -> Sta
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_observation_is_stable_across_candidate_order_and_respects_session_ownership() {
|
||||
for order in [["alpha", "beta"], ["beta", "alpha"]] {
|
||||
for (protocol, state_owner, extension_owned, body, expected) in [
|
||||
(
|
||||
"openai",
|
||||
None,
|
||||
false,
|
||||
json!({"reasoning":{"effort":"medium"}}),
|
||||
"medium",
|
||||
),
|
||||
(
|
||||
"anthropic",
|
||||
None,
|
||||
false,
|
||||
json!({"output_config":{"effort":" Future-Level "}}),
|
||||
"Future-Level",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
Some("beta"),
|
||||
false,
|
||||
json!({"previous_response_id":"resp_previous","reasoning":{"effort":"medium"}}),
|
||||
"high",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
Some("unavailable"),
|
||||
false,
|
||||
json!({"previous_response_id":"resp_previous","reasoning":{"effort":"medium"}}),
|
||||
"medium",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
Some("beta"),
|
||||
true,
|
||||
json!({"previous_response_id":"resp_previous","reasoning":{"effort":"medium"}}),
|
||||
"medium",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
Some("beta"),
|
||||
false,
|
||||
json!({"previous_response_id":"resp_previous","reasoning":{"effort":"medium"},"input":[{"type":"configuration_update","reasoning":{}}]}),
|
||||
"medium",
|
||||
),
|
||||
] {
|
||||
let kinds: Vec<_> = order
|
||||
.iter()
|
||||
.map(|name| ProviderKind::new(*name).unwrap())
|
||||
.collect();
|
||||
let directory = RuntimeAccountDirectory::new(
|
||||
kinds
|
||||
.iter()
|
||||
.map(|kind| {
|
||||
(
|
||||
ProviderAccountId::new(format!("acct_{}", kind.as_str())).unwrap(),
|
||||
RuntimeAccount::new(kind.clone(), BTreeSet::new()),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).unwrap(),
|
||||
gateway_core::settings::SettingsValues::new(
|
||||
1,
|
||||
"smart",
|
||||
Default::default(),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
kinds.clone(),
|
||||
kinds
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
ProviderModel::new(
|
||||
kind,
|
||||
UpstreamModelId::new("gpt-start").unwrap(),
|
||||
ModelCapabilities::new(BTreeSet::from([OperationKind::Generate]), None)
|
||||
.with_upstream_feature_validation(),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_start_test").unwrap(),
|
||||
PlaintextClientApiKey::new("sk_start_test").unwrap(),
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::new(directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
)
|
||||
.unwrap();
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let service =
|
||||
DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot),
|
||||
store.clone(),
|
||||
ProviderRegistry::new(order.map(|name| {
|
||||
Arc::new(SessionObservationProvider(name)) as Arc<dyn Provider>
|
||||
}))
|
||||
.unwrap(),
|
||||
Arc::new(Admissions::default()),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
Arc::new(crate::RecordingDiagnostics::default()),
|
||||
);
|
||||
let mut generate = GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(protocol, body.as_object().unwrap().clone())
|
||||
.unwrap()
|
||||
.with_context(serde_json::Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
json!("{\"request_kind\":\"review\"}"),
|
||||
)])),
|
||||
);
|
||||
if let Some(owner) = state_owner {
|
||||
let mut state = gateway_core::operation::ProviderSessionState::new(
|
||||
owner,
|
||||
serde_json::Map::from_iter([("effort".to_owned(), json!("high"))]),
|
||||
)
|
||||
.unwrap();
|
||||
if extension_owned {
|
||||
state = state.with_extension_owner(
|
||||
gateway_core::operation::ExtensionSessionOwner {
|
||||
instance_id: "plugin".into(),
|
||||
contribution_id: "adapter".into(),
|
||||
adapter_id: "adapter".into(),
|
||||
generation: 1,
|
||||
incarnation: "incarnation".into(),
|
||||
connection_local: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
generate = generate.with_provider_session_state(state);
|
||||
}
|
||||
let mut start = request(&service, ClientTransport::HttpJson);
|
||||
start.operation = Operation::Generate(generate);
|
||||
start.metadata.protocol = protocol.to_owned();
|
||||
let started = block_on(service.start(start)).expect("request observation is recorded");
|
||||
block_on(started.session.detach_finalize());
|
||||
let requests = store.requests.lock().unwrap();
|
||||
let recorded = requests.first().expect("logical request");
|
||||
assert_eq!(
|
||||
recorded.reasoning_effort.as_deref(),
|
||||
Some(expected),
|
||||
"order={order:?}, state={state_owner:?}, extension={extension_owned}"
|
||||
);
|
||||
assert_eq!(
|
||||
recorded.request_kind.as_deref(),
|
||||
(protocol == "openai").then_some("review")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SessionObservationProvider(&'static str);
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for SessionObservationProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
self.0
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
fn request_observation(
|
||||
&self,
|
||||
operation: &Operation,
|
||||
_: &ClientApiKeyId,
|
||||
) -> ProviderRequestObservation {
|
||||
ProviderRequestObservation {
|
||||
previous_reasoning_effort_override: operation
|
||||
.provider_session_state(self.0)
|
||||
.filter(|state| state.extension_owner().is_none())
|
||||
.and_then(|state| state.payload().get("effort"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
self: Arc<Self>,
|
||||
_: ProviderRequest,
|
||||
_: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_request_has_no_total_deadline_and_explicit_timeout_can_be_cleared() {
|
||||
block_on(async {
|
||||
@@ -2121,6 +2324,7 @@ impl Provider for PolicySelectingProvider {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
@@ -3239,7 +3443,7 @@ fn provider_local_probe_failure_should_remain_distinct_from_upstream() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_probe_does_not_apply_data_plane_account_model_policy() {
|
||||
fn diagnostic_probe_uses_global_profile_without_data_plane_account_model_policy() {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let account_id = ProviderAccountId::new("acct_probe").expect("account");
|
||||
let snapshot = probe_snapshot().with_account_directory(Arc::new(RuntimeAccountDirectory::new(
|
||||
@@ -3254,10 +3458,21 @@ fn diagnostic_probe_does_not_apply_data_plane_account_model_policy() {
|
||||
),
|
||||
)]),
|
||||
)));
|
||||
let profile = gateway_core::account::OpaqueProviderData::new(
|
||||
json!({"identity":"global-probe"})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
);
|
||||
let settings = snapshot
|
||||
.settings()
|
||||
.clone()
|
||||
.with_request_profiles(BTreeMap::from([(provider.clone(), profile.clone())]));
|
||||
let snapshot = snapshot.with_settings(&settings).unwrap();
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::new([Arc::new(LocalFailingProvider) as Arc<dyn Provider>])
|
||||
ProviderRegistry::new([Arc::new(ProfileCheckingProvider(profile)) as Arc<dyn Provider>])
|
||||
.expect("providers"),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
@@ -3658,6 +3873,36 @@ impl Provider for FailingProvider {
|
||||
|
||||
struct LocalFailingProvider;
|
||||
|
||||
struct ProfileCheckingProvider(gateway_core::account::OpaqueProviderData);
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for ProfileCheckingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
self: Arc<Self>,
|
||||
request: ProviderRequest,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
assert_eq!(context.request_profile(), Some(&self.0));
|
||||
Arc::new(LocalFailingProvider)
|
||||
.execute(request, context)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for LocalFailingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
|
||||
@@ -77,6 +77,7 @@ fn candidate(id: &str) -> AccountCandidate {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
|
||||
@@ -157,40 +157,28 @@ fn prompt_cache_and_image_generation_should_be_read_from_raw_body() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_detection_distinguishes_tools_from_image_inputs_and_passive_namespaces() {
|
||||
for body in [
|
||||
json!({"tools":[{"type":"image_generation"}]}),
|
||||
json!({"tool_choice":{"type":"image_generation"}}),
|
||||
json!({"tool_choice":{"type":"allowed_tools","tools":[{"type":"image_generation"}]}}),
|
||||
json!({"input":[{"type":"additional_tools","tools":[{"type":"image_generation"}]}]}),
|
||||
] {
|
||||
assert!(
|
||||
generate(body.clone()).image_generation_requested(),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
for body in [
|
||||
json!({"input":"Please generate an image"}),
|
||||
json!({"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}]}),
|
||||
json!({"tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]}),
|
||||
json!({"tools":[{"type":"function","name":"image_generation"}]}),
|
||||
] {
|
||||
assert!(
|
||||
!generate(body.clone()).image_generation_requested(),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
let private = GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"private",
|
||||
json!({"tools":[{"type":"image_generation"}]})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
fn image_generation_fact_tracks_body_replacement_and_ignores_context() {
|
||||
let operation = Operation::Generate(generate(json!({"input":"hello"})));
|
||||
let operation = operation
|
||||
.replace_protocol_wire(
|
||||
Some(Bytes::from_static(
|
||||
br#"{"tools":[{"type":"image_generation"}]}"#,
|
||||
)),
|
||||
Map::new(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(!private.image_generation_requested());
|
||||
.unwrap();
|
||||
assert!(operation.image_generation_requested());
|
||||
let operation = operation
|
||||
.replace_protocol_wire(None, Map::from_iter([("tools".to_owned(), json!([]))]))
|
||||
.unwrap();
|
||||
assert!(operation.image_generation_requested());
|
||||
let operation = operation
|
||||
.replace_protocol_wire(
|
||||
Some(Bytes::from_static(br#"{"input":"hello"}"#)),
|
||||
Map::new(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!operation.image_generation_requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -203,6 +191,10 @@ fn image_generation_fact_survives_cross_protocol_translation() {
|
||||
)
|
||||
.unwrap();
|
||||
assert!(translated.image_generation_requested());
|
||||
let replaced = translated
|
||||
.replace_protocol_wire(Some(Bytes::from_static(br#"{"messages":[]}"#)), Map::new())
|
||||
.unwrap();
|
||||
assert!(replaced.image_generation_requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -200,6 +200,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
in_flight: 2,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_reset_windows: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
@@ -251,14 +252,10 @@ fn warmup_cron_validation_and_calendar_matching() {
|
||||
] {
|
||||
assert!(!valid_warmup_cron(value), "{value:?}");
|
||||
}
|
||||
let policy = ProviderWarmupPolicy::try_new(
|
||||
true,
|
||||
"0 8,13 * * MON-FRI".to_owned(),
|
||||
Some("test-model".to_owned()),
|
||||
)
|
||||
.unwrap();
|
||||
let policy =
|
||||
ProviderWarmupPolicy::try_new(true, "0 8,13 * * MON-FRI", Some("test-model".to_owned()))
|
||||
.unwrap();
|
||||
assert!(policy.enabled());
|
||||
assert_eq!(policy.cron(), "0 8,13 * * MON-FRI");
|
||||
assert_eq!(policy.model(), Some("test-model"));
|
||||
for (day, hour, minute, expected) in [
|
||||
(9, 8, 0, true),
|
||||
@@ -274,19 +271,39 @@ fn warmup_cron_validation_and_calendar_matching() {
|
||||
.unwrap();
|
||||
assert_eq!(policy.matches_slot(slot), expected, "{slot}");
|
||||
}
|
||||
assert!(ProviderWarmupPolicy::try_new(false, String::new(), None).is_ok());
|
||||
assert!(
|
||||
ProviderWarmupPolicy::try_new(true, String::new(), Some("test-model".to_owned())).is_err()
|
||||
);
|
||||
assert!(ProviderWarmupPolicy::try_new(false, "08:00".to_owned(), None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(false, "", None).is_ok());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "", Some("test-model".to_owned())).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(false, "08:00", None).is_err());
|
||||
assert!(!valid_warmup_cron(&format!("0{}8 * * *", " ".repeat(256))));
|
||||
let leap = ProviderWarmupPolicy::try_new(false, "0 0 29 FEB *".to_owned(), None).unwrap();
|
||||
let leap = ProviderWarmupPolicy::try_new(false, "0 0 29 FEB *", None).unwrap();
|
||||
let slot = NaiveDate::from_ymd_opt(2028, 2, 29)
|
||||
.unwrap()
|
||||
.and_hms_opt(0, 0, 0)
|
||||
.unwrap();
|
||||
assert!(leap.matches_slot(slot));
|
||||
assert!(!leap.matches_slot(slot + chrono::Duration::days(1)));
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "invalid".to_owned(), None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "0 8 * * *".to_owned(), None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "invalid", None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "0 8 * * *", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_matches_only_the_scheduled_hour_and_minute_pairs() {
|
||||
use chrono::NaiveDate;
|
||||
use gateway_core::provider_ports::ProviderWarmupPolicy;
|
||||
|
||||
let policy = ProviderWarmupPolicy::try_new(
|
||||
true,
|
||||
"58 5 * * *;59 10 * * *;0 16 * * *;1 21 * * *;0 16 * * MON-FRI",
|
||||
Some("test-model".to_owned()),
|
||||
)
|
||||
.unwrap();
|
||||
let date = NaiveDate::from_ymd_opt(2026, 11, 13).unwrap();
|
||||
for minute in 0..1440 {
|
||||
let slot = date.and_hms_opt(minute / 60, minute % 60, 0).unwrap();
|
||||
assert_eq!(
|
||||
policy.matches_slot(slot),
|
||||
[358, 659, 960, 1261].contains(&minute),
|
||||
"{slot}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -877,11 +877,15 @@ fn compiler_accepts_unlimited_default_account_concurrency() {
|
||||
|
||||
#[test]
|
||||
fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
use gateway_core::account::SmartSchedulingConfig;
|
||||
use gateway_core::account::{ResetWindowWeights, SmartSchedulingConfig};
|
||||
let configs = [
|
||||
SmartSchedulingConfig::default(),
|
||||
SmartSchedulingConfig::new([0.0, 2.0, 1.0, 0.5, 1.2, 2.3]).unwrap(),
|
||||
];
|
||||
let windows = [
|
||||
ResetWindowWeights::default(),
|
||||
ResetWindowWeights::new([0.0, 1.5, 0.0]).unwrap(),
|
||||
];
|
||||
let mut plans = Vec::new();
|
||||
for (index, config) in configs.into_iter().enumerate() {
|
||||
let facts = SnapshotFacts::new(
|
||||
@@ -889,6 +893,7 @@ fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
revision(index as u64 + 1),
|
||||
SettingsValues::new(3, "smart", BTreeMap::new(), None, None)
|
||||
.with_smart_scheduling(config)
|
||||
.with_quota_reset_window_weights(windows[index])
|
||||
.with_openai_guardian_reserved_concurrency(index as u32),
|
||||
vec![],
|
||||
vec![],
|
||||
@@ -921,6 +926,10 @@ fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
index as u32
|
||||
);
|
||||
assert_eq!(plan.account_selection_policy().smart_scheduling(), config);
|
||||
assert_eq!(
|
||||
plan.account_selection_policy().quota_reset_window_weights(),
|
||||
windows[index]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user