mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 16:01:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68b297fc88 | ||
|
|
1b96c775b2 | ||
|
|
b9ccc0b702 | ||
|
|
f9e364860a | ||
|
|
ceeb68ba82 | ||
|
|
2abbfb1fac | ||
|
|
5b7f307bb2 | ||
|
|
e64fb2d955 | ||
|
|
7ee9ea8b63 | ||
|
|
ce12ba3f43 | ||
|
|
ac4ea71cd2 | ||
|
|
77de86e0b6 | ||
|
|
644296bf66 | ||
|
|
367a62bd7d | ||
|
|
62f321c82b | ||
|
|
c40e2f524e | ||
|
|
0f5bf03e13 | ||
|
|
28c8c536c5 | ||
|
|
4ee9abf9d2 | ||
|
|
779ccf4e6c | ||
|
|
d13a4a2138 | ||
|
|
a0e7f38f7f | ||
|
|
41f97ac836 | ||
|
|
975b73b33b |
@@ -125,7 +125,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
cache-bin: false
|
||||
save-if: ${{ github.event_name != 'pull_request' }}
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
|
||||
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!/frontend/.env
|
||||
|
||||
# Editors and operating systems
|
||||
/.idea/
|
||||
|
||||
Generated
+86
-54
@@ -204,7 +204,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -266,9 +266,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.10.2"
|
||||
version = "1.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a"
|
||||
checksum = "de8930c11353c72ae54aa24dc94d692df74d59814839cbaf8069d63dbcb4c519"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -292,9 +292,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.157.0"
|
||||
version = "1.159.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8"
|
||||
checksum = "8d5c431c5db2f1f6f536c2a19425575e28273859c6ab399d9ac957f73678c7db"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -318,7 +318,7 @@ dependencies = [
|
||||
"hmac",
|
||||
"http",
|
||||
"http-body",
|
||||
"lru",
|
||||
"lru 0.18.5",
|
||||
"percent-encoding",
|
||||
"regex-lite",
|
||||
"sha2",
|
||||
@@ -328,9 +328,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.4"
|
||||
version = "1.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
checksum = "57a4393e6addc3380632887b0998d06ad5f099ab5c478be0a3eeedecf8604161"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -361,9 +361,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-checksums"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307"
|
||||
checksum = "ab2f2d12a431aab3703d55277278bad45f02208569b11506220b95fdc1ef0507"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-types",
|
||||
@@ -449,18 +449,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-observability"
|
||||
version = "0.3.0"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c"
|
||||
checksum = "a416257d7b1b0f32a517a470a8c1dd354d8c4aa697ccab81a11a1e6cfb7418e3"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.2"
|
||||
version = "1.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
checksum = "5bc688673e282f42d3152dcd9cacc1a9a75734d7cd39d02e58989276fc750f3c"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -481,9 +483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.2"
|
||||
version = "1.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
checksum = "0467a2414e84b1b01e37452896fb3562b93926e81c619eba6ecf27ce9e682a0e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -509,9 +511,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "1.1.2"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9543fc722ed4865ab29fb66bd7ed0bef7b029d265ba8c587144511e30006d431"
|
||||
checksum = "f7ef30d39e131651359b60a1c560de9a9502a4c3754172ab2d6e8734fe21bcba"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -520,9 +522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.1"
|
||||
version = "1.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
checksum = "aff4bf014c512840b2bba80a20417b723cbd6a0a65c02e2d81f50b4268aedfc7"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -532,7 +534,6 @@ dependencies = [
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
"ryu",
|
||||
"serde",
|
||||
@@ -870,7 +871,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -924,7 +925,7 @@ dependencies = [
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -992,6 +993,15 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.2"
|
||||
@@ -1098,9 +1108,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.4"
|
||||
version = "1.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
|
||||
checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
@@ -1468,6 +1478,17 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a2e16b9a584b602f1e0e5e22c30ca0906786088c27500653e432d086def8b53"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.4.1"
|
||||
@@ -1511,9 +1532,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
version = "1.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide 0.9.1",
|
||||
@@ -1671,7 +1692,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1735,7 +1756,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -2526,15 +2547,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
checksum = "0e37f27df77c7887edbc5f5a59ecd8e5ec4a669319266ec1b61068667a603183"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
"data-encoding",
|
||||
"email_address",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.20.2",
|
||||
"fraction",
|
||||
"getrandom 0.3.4",
|
||||
"itoa",
|
||||
@@ -2554,18 +2575,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
checksum = "7e41151614b6c927e21982be8bef5a26741ae5a0ae6a781eedee2aa930a980e6"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
checksum = "1515b9a91134ef67d7bac95c2d76fbc7e383c4c55ee544204ceec8088f881fef"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2660,9 +2681,18 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.18.3"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c"
|
||||
checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
@@ -3308,7 +3338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3374,7 +3404,7 @@ dependencies = [
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"lru",
|
||||
"lru 0.19.0",
|
||||
"native-tls",
|
||||
"openssl",
|
||||
"openssl-sys",
|
||||
@@ -3673,14 +3703,14 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.2"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
checksum = "a028057ef852c58a42d75e09da87b5e733f4030f4c7411728955e4f87203a577"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -4043,7 +4073,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4525,9 +4555,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
version = "3.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -4625,7 +4655,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"syn 3.0.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4646,7 +4676,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"bstr",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.19.2",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
]
|
||||
@@ -5110,9 +5140,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.29.0"
|
||||
version = "1.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5715,10 +5745,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.4.0"
|
||||
version = "9.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cec733c9820347a592f6375c3c0089b479ba0a0b1cbf3dd1958b02959b7c033"
|
||||
checksum = "e3476fd497ac8af5b1ec7b51acab7bef380f7e8dd8e0487595bafd839417a302"
|
||||
dependencies = [
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
@@ -5728,13 +5759,14 @@ dependencies = [
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.6"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
|
||||
@@ -81,6 +81,7 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
"account/selection.rs" => Some(&["account", "concurrency", "identity", "validation"]),
|
||||
"concurrency.rs" => Some(&["error"]),
|
||||
"account/store.rs" => Some(&["account", "error", "identity", "validation"]),
|
||||
"account/proxy.rs" => Some(&["upstream"]),
|
||||
path if path.starts_with("policy/") => {
|
||||
Some(&["account", "identity", "policy", "validation"])
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ pub use gateway_core::account::{
|
||||
/// 账号可用的管理操作
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderAccountCapabilities {
|
||||
pub quota_limits_supported: bool,
|
||||
pub quota_settings_supported: bool,
|
||||
pub quota_credits: bool,
|
||||
pub quota: bool,
|
||||
pub quota_refresh: bool,
|
||||
@@ -113,7 +113,7 @@ pub enum AccountGroupFilter {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountRecord {
|
||||
pub quota_paused: bool,
|
||||
pub quota_limits: gateway_core::account::AccountQuotaLimits,
|
||||
pub quota_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub auto_use_quota_credits: bool,
|
||||
pub id: String,
|
||||
pub provider_kind: ProviderKind,
|
||||
@@ -254,7 +254,7 @@ pub struct AccountUpdateResult {
|
||||
/// 仅修改显式字段的批量账号设置命令
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BatchUpdateAccounts {
|
||||
pub quota_limits: Option<gateway_core::account::AccountQuotaLimits>,
|
||||
pub quota_settings: Option<gateway_core::account::AccountQuotaSettings>,
|
||||
pub auto_use_quota_credits: Option<bool>,
|
||||
pub account_ids: Vec<String>,
|
||||
pub enabled: Option<bool>,
|
||||
@@ -268,6 +268,8 @@ pub struct BatchUpdateAccounts {
|
||||
/// 已按拟提交策略重算的账号暂停结果,以快照版本防止旧判断覆盖新观测
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PreparedQuotaPause {
|
||||
pub settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub previous_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub revision: Revision,
|
||||
pub paused: bool,
|
||||
|
||||
@@ -63,6 +63,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: gateway_core::account::ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
|
||||
@@ -759,7 +759,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
self.accounts
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec![account_id.to_string()],
|
||||
enabled: Some(true),
|
||||
@@ -846,20 +846,19 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut pauses = BTreeMap::new();
|
||||
for account_id in &account_ids {
|
||||
let (mut item, provider) = self.provider_for_account(account_id).await?;
|
||||
if let Some(limits) = &command.quota_limits {
|
||||
if let Some(settings) = &command.quota_settings {
|
||||
if !provider
|
||||
.account_capabilities(account_id, &item.account.authentication_kind)
|
||||
.quota_limits_supported
|
||||
.quota_settings_supported
|
||||
{
|
||||
return Err(AdminError::invalid("该账号不支持通用额度窗口上限"));
|
||||
return Err(AdminError::invalid("该账号不支持额度桶设置"));
|
||||
}
|
||||
let new_keys = limits
|
||||
.keys()
|
||||
.filter(|key| !item.account.quota_limits.contains(key))
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>();
|
||||
if !limits.is_empty() {
|
||||
item.account.quota_limits = limits.clone();
|
||||
let previous_settings = item.account.quota_settings.clone();
|
||||
let settings = settings.clone();
|
||||
let mut observed_at = None;
|
||||
let mut paused = false;
|
||||
if !settings.is_empty() {
|
||||
item.account.quota_settings = settings.clone();
|
||||
let quota = provider
|
||||
.read_quotas(&[ProviderQuotaReadRequest {
|
||||
account: &item.account,
|
||||
@@ -870,24 +869,31 @@ impl AccountsService for DefaultAccountsService {
|
||||
.next()
|
||||
.ok_or_else(|| AdminError::invalid("缺少账号额度观测结果"))?
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
if new_keys.iter().any(|key| {
|
||||
!quota.windows.iter().any(|window| {
|
||||
window.key == *key
|
||||
&& window.local_usage_attribution
|
||||
== QuotaLocalUsageAttribution::AccountWide
|
||||
})
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能设置已观测的通用额度窗口"));
|
||||
for key in settings.keys() {
|
||||
let window = quota.windows.iter().find(|window| window.key == key);
|
||||
if window.is_none() && !previous_settings.contains(key) {
|
||||
return Err(AdminError::invalid("只能设置已观测的额度桶"));
|
||||
}
|
||||
if window.is_some_and(|window| {
|
||||
window.local_usage_attribution
|
||||
!= QuotaLocalUsageAttribution::AccountWide
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能设置通用额度桶"));
|
||||
}
|
||||
}
|
||||
pauses.insert(
|
||||
item.account.id.clone(),
|
||||
crate::model::accounts::PreparedQuotaPause {
|
||||
revision: item.account.credential_revision,
|
||||
observed_at: quota.observed_at,
|
||||
paused: !quota.paused_windows.is_empty(),
|
||||
},
|
||||
);
|
||||
observed_at = quota.observed_at;
|
||||
paused = !quota.paused_windows.is_empty();
|
||||
}
|
||||
pauses.insert(
|
||||
item.account.id.clone(),
|
||||
crate::model::accounts::PreparedQuotaPause {
|
||||
settings,
|
||||
previous_settings,
|
||||
revision: item.account.credential_revision,
|
||||
observed_at,
|
||||
paused,
|
||||
},
|
||||
);
|
||||
}
|
||||
if command.auto_use_quota_credits.is_some()
|
||||
&& !provider
|
||||
|
||||
@@ -1180,6 +1180,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 30,
|
||||
@@ -1913,7 +1914,7 @@ async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
|
||||
.batch_update(
|
||||
&context("batch-update-request"),
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
@@ -3412,7 +3413,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -51,6 +51,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -1432,6 +1432,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -257,8 +257,8 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
&["RequestLocation"][..],
|
||||
),
|
||||
(
|
||||
"../gateway-core/src/account/smart_scheduling.rs",
|
||||
&["SmartSchedulingValues", "ResetWindowWeights"][..],
|
||||
"../gateway-core/src/account/scheduling.rs",
|
||||
&["SmartSchedulingValues", "ResetWindowValues"][..],
|
||||
),
|
||||
(
|
||||
"../gateway-core/src/metering/pricing.rs",
|
||||
@@ -326,10 +326,10 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
}
|
||||
types.push(contract_type(
|
||||
item,
|
||||
if *name == "SmartSchedulingValues" {
|
||||
"SmartSchedulingConfig"
|
||||
} else {
|
||||
name
|
||||
match *name {
|
||||
"SmartSchedulingValues" => "SmartSchedulingConfig",
|
||||
"ResetWindowValues" => "ResetWindowWeights",
|
||||
_ => name,
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
@@ -343,6 +343,8 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(
|
||||
),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -117,7 +117,7 @@ pub(super) fn account_view(
|
||||
let (mut quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now, time);
|
||||
quota.protection_rules = protection_rules;
|
||||
AccountView {
|
||||
quota_limits: account.quota_limits,
|
||||
quota_settings: account.quota_settings,
|
||||
auto_use_quota_credits: account.auto_use_quota_credits,
|
||||
upstream,
|
||||
capabilities: capabilities.into(),
|
||||
|
||||
@@ -61,7 +61,7 @@ pub struct ListQuery {
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct BatchUpdateAccountsRequest {
|
||||
pub quota_limits: Option<gateway_core::account::AccountQuotaLimits>,
|
||||
pub quota_settings: Option<gateway_core::account::AccountQuotaSettings>,
|
||||
pub auto_use_quota_credits: Option<bool>,
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
pub outbound_proxy_url: Option<AccountProxyUpdate>,
|
||||
@@ -119,7 +119,7 @@ impl BatchUpdateAccountsRequest {
|
||||
.map(parse_concurrency_limit)
|
||||
.transpose()?;
|
||||
self.weight.map(parse_account_weight).transpose()?;
|
||||
if self.quota_limits.is_none()
|
||||
if self.quota_settings.is_none()
|
||||
&& self.auto_use_quota_credits.is_none()
|
||||
&& self.enabled.is_none()
|
||||
&& self.concurrency_limit.is_none()
|
||||
@@ -137,7 +137,7 @@ impl BatchUpdateAccountsRequest {
|
||||
pub(super) fn into_command(self) -> Result<BatchUpdateAccounts, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(BatchUpdateAccounts {
|
||||
quota_limits: self.quota_limits,
|
||||
quota_settings: self.quota_settings,
|
||||
auto_use_quota_credits: self.auto_use_quota_credits,
|
||||
outbound_proxy: proxy_selection(self.outbound_proxy_id, self.outbound_proxy_url)?,
|
||||
account_ids: self.account_ids,
|
||||
@@ -273,7 +273,7 @@ pub struct AccountSummaryView {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountView {
|
||||
pub quota_limits: gateway_core::account::AccountQuotaLimits,
|
||||
pub quota_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub auto_use_quota_credits: bool,
|
||||
pub upstream: Option<super::AccountUpstreamInfoView>,
|
||||
pub capabilities: AccountCapabilitiesView,
|
||||
@@ -327,7 +327,7 @@ pub struct AccountCapacityView {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountCapabilitiesView {
|
||||
pub quota_limits_supported: bool,
|
||||
pub quota_settings_supported: bool,
|
||||
pub quota_credits: bool,
|
||||
pub quota: bool,
|
||||
pub quota_refresh: bool,
|
||||
@@ -347,7 +347,7 @@ impl From<gateway_admin::model::accounts::ProviderAccountCapabilities> for Accou
|
||||
subscription: value.subscription,
|
||||
avatar: value.avatar,
|
||||
reset_credits: value.reset_credits,
|
||||
quota_limits_supported: value.quota_limits_supported,
|
||||
quota_settings_supported: value.quota_settings_supported,
|
||||
quota_credits: value.quota_credits,
|
||||
consume_reset_credit: value.consume_reset_credit,
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: gateway_core::account::ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
@@ -77,6 +78,7 @@ pub struct RuntimeSettingsView {
|
||||
pub values: RuntimeSettingsFields,
|
||||
pub config_revision: u64,
|
||||
pub smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights_defaults: gateway_core::account::ResetWindowWeights,
|
||||
pub provider_request_profiles: ProviderRequestProfiles,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub openai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
@@ -150,6 +152,8 @@ impl From<(RuntimeSettings, crate::time::TimePresenter)> for RuntimeSettingsView
|
||||
provider_request_profiles,
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights_defaults: gateway_core::account::ResetWindowWeights::default(
|
||||
),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
updated_at_display: time.datetime(&settings.updated_at),
|
||||
updated_at: settings.updated_at,
|
||||
@@ -765,6 +769,7 @@ impl RuntimeSettingsFields {
|
||||
openai_session_affinity_ttl_hours: self.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: self.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: self.smart_scheduling,
|
||||
quota_reset_window_weights: self.quota_reset_window_weights,
|
||||
min_codex_desktop_version: self.min_codex_desktop_version,
|
||||
min_codex_cli_version: self.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
@@ -808,6 +813,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
openai_session_affinity_ttl_hours: settings.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
quota_reset_window_weights: settings.quota_reset_window_weights,
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
|
||||
@@ -72,6 +72,10 @@ pub(crate) fn client_error_code(code: &str) -> &str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_retry_after_header(name: &str) -> bool {
|
||||
name.eq_ignore_ascii_case("retry-after") || name.eq_ignore_ascii_case("retry-after-ms")
|
||||
}
|
||||
|
||||
/// 为 HTTP 错误或流内失败保留原有字段,仅投影客户端需要的重试信号
|
||||
pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
const CODE_PATHS: [&str; 2] = ["/error/code", "/response/error/code"];
|
||||
@@ -87,6 +91,12 @@ pub(crate) fn capacity_error_for_client(data: &Value) -> Option<Value> {
|
||||
return None;
|
||||
}
|
||||
let mut projected = data.clone();
|
||||
// 容量等待与换号已由网关执行;旧账号的等待提示不能再次阻塞客户端的新一轮路由
|
||||
for path in ["/headers", "/error/headers", "/response/error/headers"] {
|
||||
if let Some(Value::Object(headers)) = projected.pointer_mut(path) {
|
||||
headers.retain(|name, _| !is_retry_after_header(name));
|
||||
}
|
||||
}
|
||||
for path in CODE_PATHS {
|
||||
if let Some(code) = projected.pointer_mut(path)
|
||||
&& capacity_code(code)
|
||||
|
||||
@@ -15,7 +15,7 @@ use serde_json::{Value, json};
|
||||
|
||||
pub use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client};
|
||||
use crate::client_error::{ProtocolErrorBody, capacity_error_for_client, is_retry_after_header};
|
||||
|
||||
pub use crate::client_error::{
|
||||
gateway_error_contract, gateway_error_response, openai_error_response,
|
||||
@@ -102,7 +102,9 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
/// 在下游尚未提交时交付 Provider 的 HTTP 失败响应,并应用客户端容量恢复合同
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let capacity_unavailable = matches!(error, EngineError::Provider(error)
|
||||
if error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
if error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable
|
||||
&& error.client_visible_upstream_error().and_then(|detail| detail.code())
|
||||
!= Some("previous_response_not_found"));
|
||||
if let EngineError::Provider(error) = error
|
||||
&& let Some(upstream) = error.client_visible_upstream_response()
|
||||
&& let Ok(status) = StatusCode::from_u16(upstream.status())
|
||||
@@ -110,6 +112,7 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
let projected = serde_json::from_slice::<Value>(upstream.body())
|
||||
.ok()
|
||||
.and_then(|body| capacity_error_for_client(&body));
|
||||
let suppress_retry_after = capacity_unavailable || projected.is_some();
|
||||
let mut response = match projected {
|
||||
Some(body) => {
|
||||
let mut response = Response::new(Body::from(body.to_string()));
|
||||
@@ -132,6 +135,9 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
response.headers_mut().insert(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for header in upstream.headers() {
|
||||
if suppress_retry_after && is_retry_after_header(header.name()) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
@@ -140,6 +146,13 @@ pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
if !response.headers().contains_key("x-request-id")
|
||||
&& !response.headers().contains_key("x-oai-request-id")
|
||||
&& let Some(request_id) = error.upstream_request_id()
|
||||
&& let Ok(value) = HeaderValue::from_str(request_id.as_str())
|
||||
{
|
||||
response.headers_mut().insert("x-request-id", value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
let mut response = gateway_error_response(&gateway_error_from_engine(error));
|
||||
|
||||
@@ -176,6 +176,7 @@ pub(super) fn initial_engine_error_event(
|
||||
}
|
||||
let status = if provider
|
||||
.is_some_and(|error| error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable)
|
||||
&& gateway.client_error_code() != Some("previous_response_not_found")
|
||||
{
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
} else {
|
||||
|
||||
@@ -413,7 +413,7 @@ fn account_page_item(
|
||||
protection_rules: Vec::new(),
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -580,7 +580,7 @@ mod provider {
|
||||
},
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -664,6 +664,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
.values
|
||||
.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: command.values.smart_scheduling,
|
||||
quota_reset_window_weights: command.values.quota_reset_window_weights,
|
||||
min_codex_desktop_version: command.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: command.values.min_codex_cli_version,
|
||||
usage_retention_days: command.values.usage_retention_days,
|
||||
@@ -1940,6 +1941,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
|
||||
@@ -63,6 +63,7 @@ fn update_body() -> Value {
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"quotaResetWindowWeights": gateway_core::account::ResetWindowWeights::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 31,
|
||||
@@ -264,12 +265,14 @@ fn error_protection_requires_an_explicit_policy_in_settings_replacement() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_intact() {
|
||||
async fn scheduling_settings_round_trip_and_invalid_updates_leave_the_saved_value_intact() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
let custom = json!({"loadWeight": 2.0, "quotaWeight": 0.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3, "resetWindowWeights":{"fiveHour":0.0,"weekly":1.5,"monthly":0.0}});
|
||||
body["smartScheduling"] = custom.clone();
|
||||
let windows = json!({"fiveHour":2.5,"weekly":0.0,"monthly":1.0});
|
||||
body["quotaResetWindowWeights"] = windows.clone();
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
@@ -281,6 +284,7 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = response_json(response).await;
|
||||
assert_eq!(response["data"]["smartScheduling"], custom);
|
||||
assert_eq!(response["data"]["quotaResetWindowWeights"], windows);
|
||||
assert_eq!(
|
||||
response["data"]["smartSchedulingDefaults"],
|
||||
json!(gateway_core::account::SmartSchedulingConfig::default())
|
||||
@@ -301,7 +305,7 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
invalid_values.push(missing);
|
||||
}
|
||||
for windows in [
|
||||
let invalid_windows = [
|
||||
json!({}),
|
||||
json!({"weekly":1}),
|
||||
json!(null),
|
||||
@@ -309,41 +313,53 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
json!({"fiveHour":-1,"weekly":1,"monthly":0}),
|
||||
json!({"fiveHour":0,"weekly":1,"monthly":10.1}),
|
||||
json!({"fiveHour":0,"weekly":1,"monthly":0,"daily":1}),
|
||||
] {
|
||||
];
|
||||
for windows in &invalid_windows {
|
||||
let mut invalid = custom.clone();
|
||||
invalid["resetWindowWeights"] = windows;
|
||||
invalid["resetWindowWeights"] = windows.clone();
|
||||
invalid_values.push(invalid);
|
||||
}
|
||||
for invalid in invalid_values {
|
||||
body["smartScheduling"] = invalid;
|
||||
for (field, invalid) in invalid_values
|
||||
.into_iter()
|
||||
.map(|value| ("smartScheduling", value))
|
||||
.chain(
|
||||
invalid_windows
|
||||
.into_iter()
|
||||
.map(|value| ("quotaResetWindowWeights", value)),
|
||||
)
|
||||
{
|
||||
let mut invalid_body = body.clone();
|
||||
invalid_body[field] = invalid;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body.clone()),
|
||||
Some(invalid_body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
for field in ["smartScheduling", "quotaResetWindowWeights"] {
|
||||
let mut missing = body.clone();
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(missing),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
body.as_object_mut().unwrap().remove("smartScheduling");
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["smartScheduling"],
|
||||
custom
|
||||
);
|
||||
let saved = response_json(response).await;
|
||||
assert_eq!(saved["data"]["smartScheduling"], custom);
|
||||
assert_eq!(saved["data"]["quotaResetWindowWeights"], windows);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -60,7 +60,8 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
for raw_response in [false, true] {
|
||||
let original = json!({"error": {
|
||||
"code": code, "type": "service_unavailable_error", "message": "busy",
|
||||
"param": "model", "future": {"keep": true}
|
||||
"param": "model", "future": {"keep": true},
|
||||
"headers": {"Retry-After":"309", "retry-after-ms":"309000", "x-request-id":"req_capacity"}
|
||||
}});
|
||||
let mut provider = ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
@@ -88,6 +89,10 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
Bytes::from_static(b"req_capacity"),
|
||||
),
|
||||
ProviderResponseHeader::new("retry-after", Bytes::from_static(b"7")),
|
||||
ProviderResponseHeader::new(
|
||||
"retry-after-ms",
|
||||
Bytes::from_static(b"7000"),
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
@@ -96,7 +101,8 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
if raw_response {
|
||||
assert_eq!(response.headers()["x-request-id"], "req_capacity");
|
||||
assert_eq!(response.headers()["retry-after"], "7");
|
||||
assert!(!response.headers().contains_key("retry-after"));
|
||||
assert!(!response.headers().contains_key("retry-after-ms"));
|
||||
}
|
||||
let body: Value = serde_json::from_slice(
|
||||
&to_bytes(response.into_body(), 4096)
|
||||
@@ -110,6 +116,7 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
if raw_response {
|
||||
let mut expected = original.clone();
|
||||
expected["error"]["code"] = json!("server_error");
|
||||
expected["error"]["headers"] = json!({"x-request-id":"req_capacity"});
|
||||
assert_eq!(body, expected);
|
||||
}
|
||||
let EngineError::Provider(provider) = error else {
|
||||
@@ -138,6 +145,38 @@ async fn capacity_errors_offer_client_retry_without_changing_upstream_facts() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_client_replay_preserves_recovery_status_and_failure_correlation() {
|
||||
use bytes::Bytes;
|
||||
use gateway_core::error::ClientVisibleUpstreamResponse;
|
||||
use gateway_core::upstream::OpaqueUpstreamValue;
|
||||
use std::time::Duration;
|
||||
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_retry_after(Duration::from_secs(309))
|
||||
.with_upstream_request_id(OpaqueUpstreamValue::new("req_original_capacity"))
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Resend full history.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
400,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"code":"previous_response_not_found"}}"#),
|
||||
)),
|
||||
);
|
||||
let response = engine_error_response(&error);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert_eq!(response.headers()["x-request-id"], "req_original_capacity");
|
||||
assert!(!response.headers().contains_key("retry-after"));
|
||||
assert!(!response.headers().contains_key("retry-after-ms"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn classified_capacity_error_without_special_code_returns_retryable_http_status() {
|
||||
use bytes::Bytes;
|
||||
|
||||
@@ -4,6 +4,7 @@ use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
@@ -122,10 +123,12 @@ async fn native_json_preserves_extensions_and_uses_client_key_authentication() {
|
||||
)]);
|
||||
let trace = session.trace.clone();
|
||||
let (router, execution) = fixture(session).await;
|
||||
let response = router
|
||||
.oneshot(request("/v1/messages", false))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut incoming = request("/v1/messages", false);
|
||||
incoming.headers_mut().insert(
|
||||
"x-claude-code-session-id",
|
||||
"claude-session".parse().unwrap(),
|
||||
);
|
||||
let response = router.oneshot(incoming).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(bytes, raw);
|
||||
@@ -136,6 +139,15 @@ async fn native_json_preserves_extensions_and_uses_client_key_authentication() {
|
||||
panic!("generate")
|
||||
};
|
||||
assert_eq!(generation.protocol_payload().protocol(), "anthropic");
|
||||
assert!(
|
||||
generation.protocol_payload().context()["opaque_request_headers"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!([
|
||||
"x-claude-code-session-id",
|
||||
STANDARD.encode("claude-session")
|
||||
]))
|
||||
);
|
||||
assert_eq!(
|
||||
generation.protocol_payload().body()["future_extension"]["preserved"],
|
||||
true
|
||||
|
||||
@@ -99,7 +99,7 @@ async fn fixture(sharing: ClientPeerSharing, plaintext: &str) -> (AdminTestFixtu
|
||||
projection: resolve_account_status(&facts, now.into()),
|
||||
account: AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
id: "acct_private_sentinel".into(),
|
||||
provider_kind: provider,
|
||||
|
||||
@@ -2164,13 +2164,14 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_upstream_error_event_should_be_translated_to_response_failed_for_sse() {
|
||||
for (error_type, code, client_code, kind, client_status) in [
|
||||
for (error_type, code, client_code, kind, client_status, keeps_retry_after) in [
|
||||
(
|
||||
"service_unavailable_error",
|
||||
"server_is_overloaded",
|
||||
"server_error",
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
503,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"rate_limit_error",
|
||||
@@ -2178,13 +2179,14 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
"turn_rate_limit_exceeded",
|
||||
ProviderErrorKind::RateLimited,
|
||||
429,
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error_data = json!({
|
||||
"type": "error",
|
||||
"status": 429,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"headers": {"Retry-After": "24", "retry-after-ms": "24000", "x-request-id": "req_test"},
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": code,
|
||||
@@ -2254,6 +2256,31 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
.find(|event| event.event.as_deref() == Some("response.failed"))
|
||||
.expect("translated response.failed event");
|
||||
let failed: Value = serde_json::from_str(&failed.data).expect("response.failed JSON");
|
||||
let mut expected = json!({
|
||||
"type": "response.failed",
|
||||
"status": client_status,
|
||||
"headers": {"x-request-id": "req_test"},
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": client_code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
});
|
||||
if keeps_retry_after {
|
||||
expected["headers"]["Retry-After"] = json!("24");
|
||||
expected["headers"]["retry-after-ms"] = json!("24000");
|
||||
expected["response"]["error"]["headers"] = json!({"retry-after": "24"});
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -2262,32 +2289,8 @@ async fn streaming_upstream_error_event_should_be_translated_to_response_failed_
|
||||
body.contains("event: error\n"),
|
||||
body.ends_with("data: [DONE]\n\n"),
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"status": client_status,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": client_code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true},
|
||||
"headers": {"retry-after": "24"}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
}),
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
(expected, 1, false, true),
|
||||
"{code}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1004,7 +1004,8 @@ fn capacity_failure_projection_preserves_wire_metadata_and_original_events() {
|
||||
("response.failed", false),
|
||||
] {
|
||||
for status in [None, Some(400), Some(429), Some(503)] {
|
||||
let error = json!({"code": code, "type": "service_unavailable_error", "message": "busy", "future": 42});
|
||||
let error = json!({"code": code, "type": "service_unavailable_error", "message": "busy", "future": 42,
|
||||
"headers":{"retry-after":"309", "Retry-After-Ms":"309000", "x-request-id":"req_capacity"}});
|
||||
let mut original = if event_type == "error" {
|
||||
json!({"type": event_type, "error": error, "sequence_number": 3})
|
||||
} else {
|
||||
@@ -1046,11 +1047,16 @@ fn capacity_failure_projection_preserves_wire_metadata_and_original_events() {
|
||||
assert_eq!(sse_data["response"]["error"]["code"], "server_error");
|
||||
assert_eq!(sse_data["response"]["error"]["message"], "busy");
|
||||
assert_eq!(sse_data["response"]["error"]["future"], 42);
|
||||
assert_eq!(
|
||||
sse_data["response"]["error"]["headers"],
|
||||
json!({"x-request-id":"req_capacity"})
|
||||
);
|
||||
let messages = ws.push_websocket(&event);
|
||||
let ws_data: Value = serde_json::from_str(&messages[0]).expect("JSON");
|
||||
if event_type == "error" && status.is_some() {
|
||||
let mut expected = original.clone();
|
||||
expected["error"]["code"] = json!("server_error");
|
||||
expected["error"]["headers"] = json!({"x-request-id":"req_capacity"});
|
||||
expected["status_code"] = json!(503);
|
||||
assert_eq!(ws_data, expected);
|
||||
} else {
|
||||
|
||||
@@ -285,6 +285,7 @@ async fn initial_capacity_failure_offers_client_retry_with_upstream_correlation(
|
||||
vec![
|
||||
header("x-request-id", b"req_capacity"),
|
||||
header("retry-after", b"7"),
|
||||
header("retry-after-ms", b"7000"),
|
||||
],
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
@@ -297,7 +298,8 @@ async fn initial_capacity_failure_offers_client_retry_with_upstream_correlation(
|
||||
assert_eq!(error["error"]["code"], "server_error");
|
||||
assert_eq!(error["error"]["message"], "busy");
|
||||
assert_eq!(error["headers"]["x-request-id"], "req_capacity");
|
||||
assert_eq!(error["headers"]["retry-after"], "7");
|
||||
assert!(error["headers"].get("retry-after").is_none());
|
||||
assert!(error["headers"].get("retry-after-ms").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,22 +13,28 @@ pub use model_access::{
|
||||
AccountModelAccess, AccountModelAccessMode, InvalidAccountModelAccess,
|
||||
MAX_ACCOUNT_ACCESS_MODELS,
|
||||
};
|
||||
mod quota_limits;
|
||||
pub use quota_limits::{AccountQuotaLimits, QuotaMergeBase, QuotaPauseDecision};
|
||||
mod quota_settings;
|
||||
pub use quota_settings::{
|
||||
AccountQuotaSettings, QuotaBucketSettings, QuotaBucketWarmup, QuotaMergeBase,
|
||||
QuotaPauseDecision,
|
||||
};
|
||||
mod proxy;
|
||||
pub use proxy::{InvalidOutboundProxy, OutboundProxy};
|
||||
mod scheduling;
|
||||
pub mod scope;
|
||||
mod selection;
|
||||
mod smart_scheduling;
|
||||
mod store;
|
||||
mod upstream;
|
||||
|
||||
pub use error::CredentialError;
|
||||
pub use model::*;
|
||||
pub use scheduling::{
|
||||
QuotaResetPeriod, ResetWindowWeights, SchedulingConfigError, SmartSchedulingConfig,
|
||||
};
|
||||
pub(crate) use selection::smart_score;
|
||||
pub use selection::*;
|
||||
pub use smart_scheduling::{QuotaResetPeriod, SmartSchedulingConfig, SmartSchedulingConfigError};
|
||||
pub use store::{
|
||||
AccountCatalogObservation, AccountCatalogOmissions, ProviderAccountStore, StoredAccountCatalog,
|
||||
AccountCatalogObservation, AccountCatalogOmissions, ProviderAccountStore, QuotaWarmupState,
|
||||
QuotaWarmupStateUpdate, StoredAccountCatalog,
|
||||
};
|
||||
pub use upstream::*;
|
||||
|
||||
@@ -828,7 +828,7 @@ pub struct ProviderAccount {
|
||||
enabled: bool,
|
||||
concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
weight: AccountWeight,
|
||||
quota_limits: super::AccountQuotaLimits,
|
||||
quota_settings: super::AccountQuotaSettings,
|
||||
quota_paused: bool,
|
||||
auto_use_quota_credits: bool,
|
||||
model_access: super::AccountModelAccess,
|
||||
@@ -869,7 +869,7 @@ impl ProviderAccount {
|
||||
enabled: true,
|
||||
concurrency_limit: None,
|
||||
weight: AccountWeight::DEFAULT,
|
||||
quota_limits: super::AccountQuotaLimits::disabled(),
|
||||
quota_settings: super::AccountQuotaSettings::disabled(),
|
||||
quota_paused: false,
|
||||
auto_use_quota_credits: false,
|
||||
model_access: super::AccountModelAccess::all(),
|
||||
@@ -886,16 +886,20 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_quota_limits(mut self, limits: super::AccountQuotaLimits, paused: bool) -> Self {
|
||||
pub fn with_quota_settings(
|
||||
mut self,
|
||||
limits: super::AccountQuotaSettings,
|
||||
paused: bool,
|
||||
) -> Self {
|
||||
// 只有限额账号需要在取得租约前检测额度窗口的并发更新
|
||||
self.quota_paused = paused;
|
||||
self.quota_limits = limits;
|
||||
self.quota_settings = limits;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota_limits(&self) -> &super::AccountQuotaLimits {
|
||||
&self.quota_limits
|
||||
pub const fn quota_settings(&self) -> &super::AccountQuotaSettings {
|
||||
&self.quota_settings
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
use std::fmt;
|
||||
use url::Url;
|
||||
|
||||
use crate::upstream::ProxyEndpoint;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct OutboundProxy {
|
||||
url: Url,
|
||||
@@ -51,6 +53,26 @@ impl OutboundProxy {
|
||||
self.url.as_str()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn dial_endpoint(&self) -> ProxyEndpoint {
|
||||
ProxyEndpoint::new(
|
||||
self.url.host_str().expect("validated proxy host"),
|
||||
self.url
|
||||
.port_or_known_default()
|
||||
.expect("validated proxy port"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn matches_endpoint(&self, endpoint: &ProxyEndpoint) -> bool {
|
||||
endpoint.matches(
|
||||
self.url.host_str().expect("validated proxy host"),
|
||||
self.url
|
||||
.port_or_known_default()
|
||||
.expect("validated proxy port"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn endpoint(&self) -> String {
|
||||
let mut url = self.url.clone();
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
//! 管理员按账号通用窗口设置的已用比例上限
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(try_from = "BTreeMap<String, u8>")]
|
||||
pub struct AccountQuotaLimits(BTreeMap<String, u8>);
|
||||
|
||||
impl AccountQuotaLimits {
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self(BTreeMap::new())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.0.contains_key(key)
|
||||
}
|
||||
|
||||
pub fn keys(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.keys().map(String::as_str)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reached(&self, key: &str, used: Option<f64>) -> bool {
|
||||
self.0
|
||||
.get(key)
|
||||
.zip(used)
|
||||
.is_some_and(|(&limit, used)| used >= f64::from(limit))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<BTreeMap<String, u8>> for AccountQuotaLimits {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from(limits: BTreeMap<String, u8>) -> Result<Self, Self::Error> {
|
||||
if limits.iter().any(|(key, limit)| {
|
||||
key.is_empty()
|
||||
|| key.len() > 256
|
||||
|| key.chars().any(char::is_control)
|
||||
|| !(1..=100).contains(limit)
|
||||
}) {
|
||||
return Err("额度窗口标识无效或上限不在 1~100 范围内");
|
||||
}
|
||||
Ok(Self(limits))
|
||||
}
|
||||
}
|
||||
|
||||
/// 合并时读到的快照版本,时间为空表示当时尚无快照
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaMergeBase {
|
||||
pub observed_at: Option<std::time::SystemTime>,
|
||||
}
|
||||
|
||||
/// 此次判断依据的策略,以及是否有通用窗口触发暂停
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct QuotaPauseDecision {
|
||||
/// 独立观测无需合并,依赖旧快照的判断必须随写入校验版本
|
||||
pub merge_base: Option<QuotaMergeBase>,
|
||||
pub limits: AccountQuotaLimits,
|
||||
pub paused: bool,
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! 管理员按额度窗口设置的使用上限与到期预热策略
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(try_from = "BTreeMap<String, QuotaBucketSettings>")]
|
||||
pub struct AccountQuotaSettings(BTreeMap<String, QuotaBucketSettings>);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct QuotaBucketSettings {
|
||||
pub limit_percent: Option<u8>,
|
||||
pub warmup: Option<QuotaBucketWarmup>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct QuotaBucketWarmup {
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl AccountQuotaSettings {
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self(BTreeMap::new())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = (&str, &QuotaBucketSettings)> {
|
||||
self.0
|
||||
.iter()
|
||||
.map(|(key, settings)| (key.as_str(), settings))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_warmup(&self) -> bool {
|
||||
self.0.values().any(|settings| settings.warmup.is_some())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.0.contains_key(key)
|
||||
}
|
||||
|
||||
pub fn keys(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.keys().map(String::as_str)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reached(&self, key: &str, used: Option<f64>) -> bool {
|
||||
self.0
|
||||
.get(key)
|
||||
.and_then(|settings| settings.limit_percent)
|
||||
.zip(used)
|
||||
.is_some_and(|(limit, used)| used >= f64::from(limit))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<BTreeMap<String, QuotaBucketSettings>> for AccountQuotaSettings {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from(limits: BTreeMap<String, QuotaBucketSettings>) -> Result<Self, Self::Error> {
|
||||
if limits.iter().any(|(key, settings)| {
|
||||
key.is_empty()
|
||||
|| key.len() > 256
|
||||
|| key.chars().any(char::is_control)
|
||||
|| settings
|
||||
.limit_percent
|
||||
.is_some_and(|limit| !(1..=100).contains(&limit))
|
||||
|| settings.warmup.as_ref().is_some_and(|warmup| {
|
||||
warmup.model.is_empty()
|
||||
|| warmup.model.len() > 128
|
||||
|| warmup.model.trim() != warmup.model
|
||||
|| warmup.model.chars().any(char::is_control)
|
||||
})
|
||||
}) {
|
||||
return Err("额度桶设置无效");
|
||||
}
|
||||
Ok(Self(limits))
|
||||
}
|
||||
}
|
||||
|
||||
/// 合并时读到的快照版本,时间为空表示当时尚无快照
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaMergeBase {
|
||||
pub observed_at: Option<std::time::SystemTime>,
|
||||
}
|
||||
|
||||
/// 此次判断依据的策略,以及是否有通用窗口触发暂停
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct QuotaPauseDecision {
|
||||
/// 独立观测无需合并,依赖旧快照的判断必须随写入校验版本
|
||||
pub merge_base: Option<QuotaMergeBase>,
|
||||
pub limits: AccountQuotaSettings,
|
||||
pub paused: bool,
|
||||
}
|
||||
+88
-44
@@ -1,4 +1,4 @@
|
||||
//! 智能调度参数;固定十分位保证快照可精确比较,浮点数只用于 wire 与评分
|
||||
//! 账号调度参数与共享重置窗口评分;固定十分位保证快照可精确比较,浮点数只用于 wire 与评分
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
|
||||
#[serde(into = "SmartSchedulingValues")]
|
||||
pub struct SmartSchedulingConfig {
|
||||
weights: [u8; 6],
|
||||
reset_windows: [u8; 3],
|
||||
reset_windows: ResetWindowWeights,
|
||||
}
|
||||
|
||||
/// 账号通用额度的周期;Provider 按上游周期事实投影,不依赖窗口名称和槽位
|
||||
@@ -30,9 +30,9 @@ impl QuotaResetPeriod {
|
||||
}
|
||||
}
|
||||
|
||||
fn weight_tenths(value: f64, field: &'static str) -> Result<u8, SmartSchedulingConfigError> {
|
||||
fn weight_tenths(value: f64, field: &'static str) -> Result<u8, SchedulingConfigError> {
|
||||
if !(0.0..=10.0).contains(&value) || (value * 10.0).round() / 10.0 != value {
|
||||
return Err(SmartSchedulingConfigError::InvalidWeight(field));
|
||||
return Err(SchedulingConfigError::InvalidWeight(field));
|
||||
}
|
||||
Ok((value * 10.0).round() as u8)
|
||||
}
|
||||
@@ -52,13 +52,13 @@ impl Default for SmartSchedulingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
weights: [10, 8, 10, 5, 0, 0],
|
||||
reset_windows: [0; 3],
|
||||
reset_windows: ResetWindowWeights::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum SmartSchedulingConfigError {
|
||||
pub enum SchedulingConfigError {
|
||||
#[error("{0} must be between 0 and 10 with at most one decimal place")]
|
||||
InvalidWeight(&'static str),
|
||||
#[error("at least one smart scheduling weight must be positive")]
|
||||
@@ -67,7 +67,7 @@ pub enum SmartSchedulingConfigError {
|
||||
|
||||
impl SmartSchedulingConfig {
|
||||
/// 顺序为负载、剩余额度、健康、首输出延迟、额度重置和排队压力
|
||||
pub fn new(weights: [f64; 6]) -> Result<Self, SmartSchedulingConfigError> {
|
||||
pub fn new(weights: [f64; 6]) -> Result<Self, SchedulingConfigError> {
|
||||
let mut tenths = [0; 6];
|
||||
for (index, field) in [
|
||||
"loadWeight",
|
||||
@@ -83,7 +83,7 @@ impl SmartSchedulingConfig {
|
||||
tenths[index] = weight_tenths(weights[index], field)?;
|
||||
}
|
||||
if tenths == [0; 6] {
|
||||
return Err(SmartSchedulingConfigError::EmptyWeights);
|
||||
return Err(SchedulingConfigError::EmptyWeights);
|
||||
}
|
||||
Ok(Self {
|
||||
weights: tenths,
|
||||
@@ -97,35 +97,14 @@ impl SmartSchedulingConfig {
|
||||
}
|
||||
|
||||
fn reset_enabled(self) -> bool {
|
||||
self.weights[4] > 0 && self.reset_windows != [0; 3]
|
||||
self.weights[4] > 0 && self.reset_windows.enabled()
|
||||
}
|
||||
|
||||
pub(crate) fn reset_score(self, signals: &AccountRuntimeSignals, now: SystemTime) -> f64 {
|
||||
if !self.reset_enabled() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut scores = [0.0_f64; 3];
|
||||
for window in signals.quota_reset_windows.as_deref().unwrap_or_default() {
|
||||
let Ok(remaining) = window.reset_at.duration_since(now) else {
|
||||
continue;
|
||||
};
|
||||
if remaining.is_zero() {
|
||||
continue;
|
||||
}
|
||||
let half = f64::from(window.period.seconds()) / 7.0;
|
||||
let score = half / (half + remaining.as_secs_f64());
|
||||
// 同一周期只计最近的有效重置,避免重复额度桶叠加得分
|
||||
let slot = &mut scores[window.period as usize];
|
||||
*slot = slot.max(score);
|
||||
}
|
||||
// 缺失周期得零分,分母固定,避免少报窗口反而提高排名
|
||||
let weights = self.reset_windows.map(f64::from);
|
||||
scores
|
||||
.into_iter()
|
||||
.zip(weights)
|
||||
.map(|(score, w)| score * w)
|
||||
.sum::<f64>()
|
||||
/ weights.into_iter().sum::<f64>()
|
||||
self.reset_windows.score(signals, now)
|
||||
}
|
||||
|
||||
pub(crate) fn score_tolerance(self) -> f64 {
|
||||
@@ -157,14 +136,14 @@ struct SmartSchedulingValues {
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct ResetWindowWeights {
|
||||
struct ResetWindowValues {
|
||||
five_hour: f64,
|
||||
weekly: f64,
|
||||
monthly: f64,
|
||||
}
|
||||
|
||||
impl TryFrom<SmartSchedulingValues> for SmartSchedulingConfig {
|
||||
type Error = SmartSchedulingConfigError;
|
||||
type Error = SchedulingConfigError;
|
||||
|
||||
fn try_from(value: SmartSchedulingValues) -> Result<Self, Self::Error> {
|
||||
let mut config = Self::new([
|
||||
@@ -175,12 +154,7 @@ impl TryFrom<SmartSchedulingValues> for SmartSchedulingConfig {
|
||||
value.reset_weight,
|
||||
value.queue_weight,
|
||||
])?;
|
||||
let windows = value.reset_window_weights;
|
||||
config.reset_windows = [
|
||||
weight_tenths(windows.five_hour, "fiveHour")?,
|
||||
weight_tenths(windows.weekly, "weekly")?,
|
||||
weight_tenths(windows.monthly, "monthly")?,
|
||||
];
|
||||
config.reset_windows = value.reset_window_weights;
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
@@ -202,11 +176,81 @@ impl From<SmartSchedulingConfig> for SmartSchedulingValues {
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
reset_window_weights: ResetWindowWeights {
|
||||
five_hour: f64::from(value.reset_windows[0]) / 10.0,
|
||||
weekly: f64::from(value.reset_windows[1]) / 10.0,
|
||||
monthly: f64::from(value.reset_windows[2]) / 10.0,
|
||||
},
|
||||
reset_window_weights: value.reset_windows,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 两种调度策略共用的重置窗口系数,各策略独立持有配置
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(into = "ResetWindowValues")]
|
||||
pub struct ResetWindowWeights {
|
||||
weights: [u8; 3],
|
||||
}
|
||||
|
||||
impl ResetWindowWeights {
|
||||
/// 顺序为 5 小时、周、月;全零表示未指定窗口
|
||||
pub fn new(weights: [f64; 3]) -> Result<Self, SchedulingConfigError> {
|
||||
Ok(Self {
|
||||
weights: [
|
||||
weight_tenths(weights[0], "fiveHour")?,
|
||||
weight_tenths(weights[1], "weekly")?,
|
||||
weight_tenths(weights[2], "monthly")?,
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn enabled(self) -> bool {
|
||||
self.weights != [0; 3]
|
||||
}
|
||||
|
||||
pub(crate) fn score(self, signals: &AccountRuntimeSignals, now: SystemTime) -> f64 {
|
||||
if !self.enabled() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut scores = [0.0_f64; 3];
|
||||
for window in signals.quota_reset_windows.as_deref().unwrap_or_default() {
|
||||
let Ok(remaining) = window.reset_at.duration_since(now) else {
|
||||
continue;
|
||||
};
|
||||
if remaining.is_zero() {
|
||||
continue;
|
||||
}
|
||||
let half = f64::from(window.period.seconds()) / 7.0;
|
||||
let score = half / (half + remaining.as_secs_f64());
|
||||
// 同一周期只计最近的有效重置,避免重复额度桶叠加得分
|
||||
let slot = &mut scores[window.period as usize];
|
||||
*slot = slot.max(score);
|
||||
}
|
||||
// 缺失周期得零分,分母固定,避免少报窗口反而提高排名
|
||||
let weights = self.weights.map(f64::from);
|
||||
scores
|
||||
.into_iter()
|
||||
.zip(weights)
|
||||
.map(|(score, w)| score * w)
|
||||
.sum::<f64>()
|
||||
/ weights.into_iter().sum::<f64>()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for ResetWindowWeights {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// 与智能调度一致,先恢复 flatten 缓冲的小数再校验
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let values =
|
||||
serde_json::from_value::<ResetWindowValues>(value).map_err(serde::de::Error::custom)?;
|
||||
Self::new([values.five_hour, values.weekly, values.monthly])
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ResetWindowWeights> for ResetWindowValues {
|
||||
fn from(value: ResetWindowWeights) -> Self {
|
||||
let [five_hour, weekly, monthly] = value.weights.map(|weight| f64::from(weight) / 10.0);
|
||||
Self {
|
||||
five_hour,
|
||||
weekly,
|
||||
monthly,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,8 @@ use crate::concurrency::{CapacityWait, ConcurrencyQueuePolicy, QueueRejection};
|
||||
use crate::identity::ProviderKind;
|
||||
|
||||
use super::{
|
||||
AccountConcurrency, AccountStatus, ProviderAccount, ProviderAccountId, SmartSchedulingConfig,
|
||||
AccountConcurrency, AccountStatus, ProviderAccount, ProviderAccountId, ResetWindowWeights,
|
||||
SmartSchedulingConfig,
|
||||
};
|
||||
|
||||
/// `runtime_settings.rotation_strategy` 的稳定值
|
||||
@@ -90,6 +91,7 @@ impl AccountAffinity {
|
||||
pub struct AccountSelectionPolicy {
|
||||
strategy: RotationStrategy,
|
||||
smart_scheduling: SmartSchedulingConfig,
|
||||
quota_reset_window_weights: ResetWindowWeights,
|
||||
max_concurrent_per_account: AccountConcurrency,
|
||||
queue_policy: ConcurrencyQueuePolicy,
|
||||
openai_guardian_reserved_concurrency: u32,
|
||||
@@ -108,6 +110,7 @@ impl AccountSelectionPolicy {
|
||||
Self {
|
||||
strategy,
|
||||
smart_scheduling: SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: ResetWindowWeights::default(),
|
||||
max_concurrent_per_account: max_concurrent_per_account.into(),
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: AccountAffinity::default(),
|
||||
@@ -121,6 +124,17 @@ impl AccountSelectionPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_quota_reset_window_weights(mut self, weights: ResetWindowWeights) -> Self {
|
||||
self.quota_reset_window_weights = weights;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota_reset_window_weights(self) -> ResetWindowWeights {
|
||||
self.quota_reset_window_weights
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_account_affinity(mut self, affinity: AccountAffinity) -> Self {
|
||||
self.openai_account_affinity = affinity;
|
||||
@@ -799,24 +813,31 @@ impl AccountSelector {
|
||||
let candidate = match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
eligible.sort_by(|left, right| {
|
||||
(
|
||||
left.signals.quota_reset_at.is_none(),
|
||||
left.signals.quota_reset_at,
|
||||
)
|
||||
.cmp(&(
|
||||
right.signals.quota_reset_at.is_none(),
|
||||
right.signals.quota_reset_at,
|
||||
))
|
||||
.then_with(|| {
|
||||
capacity_utilization(left, context)
|
||||
.total_cmp(&capacity_utilization(right, context))
|
||||
})
|
||||
.then_with(|| {
|
||||
left.signals
|
||||
.last_started_at
|
||||
.cmp(&right.signals.last_started_at)
|
||||
})
|
||||
.then_with(|| left.account.id().cmp(right.account.id()))
|
||||
let windows = context.policy.quota_reset_window_weights();
|
||||
if windows.enabled() {
|
||||
windows
|
||||
.score(&right.signals, context.now)
|
||||
.total_cmp(&windows.score(&left.signals, context.now))
|
||||
} else {
|
||||
(
|
||||
left.signals.quota_reset_at.is_none(),
|
||||
left.signals.quota_reset_at,
|
||||
)
|
||||
.cmp(&(
|
||||
right.signals.quota_reset_at.is_none(),
|
||||
right.signals.quota_reset_at,
|
||||
))
|
||||
}
|
||||
.then_with(|| {
|
||||
capacity_utilization(left, context)
|
||||
.total_cmp(&capacity_utilization(right, context))
|
||||
})
|
||||
.then_with(|| {
|
||||
left.signals
|
||||
.last_started_at
|
||||
.cmp(&right.signals.last_started_at)
|
||||
})
|
||||
.then_with(|| left.account.id().cmp(right.account.id()))
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
|
||||
@@ -35,9 +35,38 @@ pub struct AccountCatalogOmissions {
|
||||
pub as_of: std::time::SystemTime,
|
||||
}
|
||||
|
||||
/// Provider 自行解释预热进度;版本只用于并发领取和观测合并
|
||||
#[derive(Clone, Default)]
|
||||
pub struct QuotaWarmupState {
|
||||
pub version: u64,
|
||||
pub data: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
pub struct QuotaWarmupStateUpdate<'a> {
|
||||
pub account: &'a ProviderAccount,
|
||||
pub expected_version: u64,
|
||||
pub expected_quota_observed_at: Option<std::time::SystemTime>,
|
||||
pub data: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口
|
||||
#[async_trait]
|
||||
pub trait ProviderAccountStore: Send + Sync {
|
||||
async fn load_quota_warmup_state(
|
||||
&self,
|
||||
_account: &ProviderAccount,
|
||||
) -> Result<Option<QuotaWarmupState>, StoreError> {
|
||||
Err(StoreError::new(crate::error::StoreErrorKind::InvalidData))
|
||||
}
|
||||
|
||||
/// 配置、凭据或进度变化均使领取失败,取消或崩溃后不能重放已领取的请求
|
||||
async fn compare_and_swap_quota_warmup_state(
|
||||
&self,
|
||||
_update: QuotaWarmupStateUpdate<'_>,
|
||||
) -> Result<bool, StoreError> {
|
||||
Err(StoreError::new(crate::error::StoreErrorKind::InvalidData))
|
||||
}
|
||||
|
||||
async fn load_account_catalogs(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
|
||||
@@ -367,6 +367,7 @@ where
|
||||
account_rotations: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
pending_retry: None,
|
||||
@@ -525,6 +526,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
account_rotations: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
excluded_proxy_endpoints: Vec<crate::upstream::ProxyEndpoint>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算
|
||||
@@ -1019,6 +1021,10 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn has_account_rotation_budget(&self) -> bool {
|
||||
self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
let pending_retry = self.pending_retry.take();
|
||||
let is_transport_recovery = pending_retry
|
||||
@@ -1094,7 +1100,7 @@ where
|
||||
),
|
||||
}
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
let mut account_context = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(target) => AccountAttemptContext::diagnostic(
|
||||
self.excluded_accounts.clone(),
|
||||
target.clone(),
|
||||
@@ -1111,6 +1117,9 @@ where
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
}));
|
||||
account_context
|
||||
.excluded_proxy_endpoints
|
||||
.clone_from(&self.excluded_proxy_endpoints);
|
||||
let provider = self
|
||||
.engine
|
||||
.providers()
|
||||
@@ -1230,9 +1239,23 @@ where
|
||||
}
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
Err(mut error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
if self.continuation_attempt == ContinuationAttempt::RotateForClientReplay
|
||||
&& error.continuation_recovery_disposition()
|
||||
== Some(ContinuationRecoveryDisposition::ClientReplayRequired)
|
||||
&& let Some(original) = self.last_retryable_failure.take()
|
||||
{
|
||||
// 迁移只改变客户端恢复指令;失败分类、发送事实与运维正文仍属于原请求
|
||||
error = original.with_client_recovery_from(error);
|
||||
self.last_retryable_failure_events.clear();
|
||||
}
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& self
|
||||
.last_retryable_failure
|
||||
.as_ref()
|
||||
.and_then(ProviderError::unreachable_proxy)
|
||||
.is_none()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
@@ -1264,7 +1287,7 @@ where
|
||||
)
|
||||
// 跨 Provider 候选推进必然换号(账号行按 Provider 隔离),
|
||||
// 预算耗尽后不再推进,交回容量类失败的原有终态语义。
|
||||
&& self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
|
||||
&& self.has_account_rotation_budget()
|
||||
&& self.advance_provider_candidate();
|
||||
let retryable = self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
@@ -1668,10 +1691,8 @@ where
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
// 只有尚未发送的逻辑请求进入新增策略;已有发送后安全恢复保持原有路由规则
|
||||
let connection_retry_requested = self.current_send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& error.allows_connection_rotation();
|
||||
// 服务器建议不可截短
|
||||
// 超出上限,或上游拒绝继续重试当前账号时,跳过同账号等待
|
||||
// 较短的 Retry-After 也直接沿用安全换号或原错误出口
|
||||
@@ -1729,9 +1750,8 @@ where
|
||||
let can_rotate_connection = connection_retry_requested
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
|
||||
&& self.has_account_rotation_budget();
|
||||
// A 先恢复一次,再利用预留预算重选;下一次传输仍由 Provider 的恢复策略决定
|
||||
let connection_rotation = transport_recovery
|
||||
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
|
||||
let reserve_connection_rotation = can_rotate_connection
|
||||
@@ -1808,8 +1828,7 @@ where
|
||||
&& !is_return_error_retry(&error)
|
||||
&& transport_recovery.is_none()
|
||||
&& (ordinary_retry || account_rotation_retry)
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
&& self.has_account_rotation_budget();
|
||||
let retryable = !error.retry_is_prohibited()
|
||||
&& (continuation_retry
|
||||
|| same_account_retry
|
||||
@@ -2000,6 +2019,11 @@ where
|
||||
}
|
||||
StreamRetry::Pending(retry) => self.pending_retry = Some(retry),
|
||||
StreamRetry::ConnectionRotation(retry) => {
|
||||
if let Some(endpoint) = pending.error.unreachable_proxy()
|
||||
&& !self.excluded_proxy_endpoints.contains(endpoint)
|
||||
{
|
||||
self.excluded_proxy_endpoints.push(endpoint.clone());
|
||||
}
|
||||
self.excluded_accounts.insert(retry.account.clone());
|
||||
self.connection_rotation = Some(retry);
|
||||
}
|
||||
@@ -2186,6 +2210,26 @@ where
|
||||
}
|
||||
|
||||
match self.continuation_attempt {
|
||||
ContinuationAttempt::Native
|
||||
if error.continuation_recovery_disposition()
|
||||
== Some(ContinuationRecoveryDisposition::ClientReplayRequired)
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransientRetry {
|
||||
policy: crate::error::TransientRetryPolicy::RetryThenRotateAccount,
|
||||
..
|
||||
})
|
||||
)
|
||||
&& error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after())
|
||||
&& !error.retry_is_prohibited()
|
||||
&& self.has_account_rotation_budget() =>
|
||||
{
|
||||
self.continuation_attempt = ContinuationAttempt::RotateForClientReplay;
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
ContinuationAttempt::Native => match error.continuation_recovery_disposition() {
|
||||
Some(ContinuationRecoveryDisposition::RetryExactConnection) => {
|
||||
self.recovery_account = Some(current.metadata.provider_account_id().clone());
|
||||
@@ -2209,8 +2253,7 @@ where
|
||||
// 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放,
|
||||
// 落回不可重试路径以原始上游错误终态。
|
||||
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny
|
||||
if self.account_rotations
|
||||
>= self.plan.account_selection_policy().max_account_rotations() =>
|
||||
if !self.has_account_rotation_budget() =>
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -2223,7 +2266,7 @@ where
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
ContinuationAttempt::None => return false,
|
||||
ContinuationAttempt::None | ContinuationAttempt::RotateForClientReplay => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -2262,7 +2305,7 @@ where
|
||||
return false;
|
||||
}
|
||||
// Native 期间仍可能配置为禁止换号,跨账号重放共用请求冻结的预算
|
||||
if self.account_rotations >= self.plan.account_selection_policy().max_account_rotations() {
|
||||
if !self.has_account_rotation_budget() {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +124,7 @@ impl AttemptTrigger {
|
||||
/// 一旦设置,Provider 在本 attempt 内不得换号或切换 target
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccountAttemptContext {
|
||||
excluded_proxy_endpoints: Vec<crate::upstream::ProxyEndpoint>,
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: Option<ProviderAccountId>,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
@@ -140,6 +141,7 @@ impl AccountAttemptContext {
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account,
|
||||
state_owner,
|
||||
@@ -160,6 +162,7 @@ impl AccountAttemptContext {
|
||||
) -> Self {
|
||||
let (required_account, eligibility) = target.into_selection();
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account: Some(required_account),
|
||||
state_owner,
|
||||
@@ -265,6 +268,8 @@ pub enum ContinuationAttempt {
|
||||
///
|
||||
/// 该枚举本身不证明客户端 input 完整,也不授权 Provider 清除 previous-response
|
||||
ReplayAny,
|
||||
/// 仅迁移账号绑定,随后要求客户端携带完整历史重放;不得发送当前增量输入
|
||||
RotateForClientReplay,
|
||||
}
|
||||
|
||||
impl ContinuationAttempt {
|
||||
@@ -275,6 +280,7 @@ impl ContinuationAttempt {
|
||||
Self::Native => "native",
|
||||
Self::ReplayOwner => "replay_owner",
|
||||
Self::ReplayAny => "replay_any",
|
||||
Self::RotateForClientReplay => "rotate_for_client_replay",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -593,7 +599,9 @@ impl AttemptContext {
|
||||
.account_state_owner()
|
||||
.filter(|owner| owner.provider() == provider)
|
||||
.map(|owner| owner.account()),
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny => None,
|
||||
ContinuationAttempt::None
|
||||
| ContinuationAttempt::ReplayAny
|
||||
| ContinuationAttempt::RotateForClientReplay => None,
|
||||
});
|
||||
self.request.upstream_adapters.as_ref().map_or(
|
||||
Ok(upstream_adapter::UpstreamAdapterSelection::Native),
|
||||
@@ -896,6 +904,17 @@ impl AttemptContext {
|
||||
self.account.excluded_accounts()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn excludes_proxy(&self, proxy: Option<&crate::account::OutboundProxy>) -> bool {
|
||||
!self.account.excluded_proxy_endpoints.is_empty()
|
||||
&& proxy.is_some_and(|proxy| {
|
||||
self.account
|
||||
.excluded_proxy_endpoints
|
||||
.iter()
|
||||
.any(|endpoint| proxy.matches_endpoint(endpoint))
|
||||
})
|
||||
}
|
||||
|
||||
/// 管理端诊断或请求局部恢复 attempt 强制使用的唯一账号
|
||||
#[must_use]
|
||||
pub const fn required_account(&self) -> Option<&ProviderAccountId> {
|
||||
|
||||
@@ -117,7 +117,8 @@ pub enum ContinuationFailure {
|
||||
pub enum ContinuationRecoveryDisposition {
|
||||
/// 只能再次取得同账号、同一精确连接,不能改变 continuation scope
|
||||
RetryExactConnection,
|
||||
/// 代理停止内部恢复,由客户端携带完整输入创建新链
|
||||
/// 当前增量不能由代理重放,由客户端携带完整输入创建新链
|
||||
/// 显式换号策略仍可迁移会话绑定,但不能发送增量输入
|
||||
ClientReplayRequired,
|
||||
/// Provider 已证明可以进入 owner 或其他合法目标的恢复流程
|
||||
ProviderReplayAllowed,
|
||||
@@ -135,6 +136,7 @@ pub enum TransientRetryPolicy {
|
||||
}
|
||||
|
||||
/// Provider 在下游尚未提交事件时请求的隐藏恢复方式
|
||||
/// 传输恢复默认保持原账号;明确允许建连换号时,Core 可在同账号恢复失败后重选
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum PreDeliveryRetry {
|
||||
/// 排除本次账号,按普通调度策略重新选号
|
||||
@@ -152,7 +154,7 @@ pub enum PreDeliveryRetry {
|
||||
/// 同账号恢复策略,不提供额外安全重放资格
|
||||
policy: TransientRetryPolicy,
|
||||
},
|
||||
/// 固定本次账号,并按 Provider 给出的序号重试当前传输
|
||||
/// 按 Provider 给出的序号重试当前传输
|
||||
/// 可用于发送结果不明确时的有界尽力恢复,不等于上游未执行的证明
|
||||
SameAccountTransportRetry {
|
||||
/// Provider-owned 传输重试序号,从 1 开始
|
||||
@@ -165,7 +167,7 @@ pub enum PreDeliveryRetry {
|
||||
/// Provider 明确指定实际失败的传输,避免 HTTP 恢复重新进入 WS
|
||||
transport: crate::upstream::AttemptTransport,
|
||||
},
|
||||
/// 固定本次账号,并要求 Provider 使用备用传输
|
||||
/// 要求 Provider 使用备用传输
|
||||
SameAccountTransportFallback,
|
||||
}
|
||||
|
||||
@@ -517,6 +519,7 @@ pub struct ProviderError {
|
||||
replay_safe: bool,
|
||||
retry_prohibited: bool,
|
||||
same_account_retry_blocked: bool,
|
||||
connection_rotation_allowed: bool,
|
||||
pre_delivery_retry: Option<Box<PreDeliveryRetry>>,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
@@ -538,6 +541,7 @@ struct ProviderErrorUpstreamValues {
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorFailureObservation {
|
||||
unreachable_proxy: Option<crate::upstream::ProxyEndpoint>,
|
||||
continuation_unavailable_reason: Option<&'static str>,
|
||||
connection: Option<ProviderConnectionObservation>,
|
||||
}
|
||||
@@ -561,6 +565,7 @@ impl ProviderError {
|
||||
replay_safe: false,
|
||||
retry_prohibited: false,
|
||||
same_account_retry_blocked: false,
|
||||
connection_rotation_allowed: false,
|
||||
pre_delivery_retry: None,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
@@ -659,6 +664,20 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 仅当连接账号绑定代理的入口失败时记录;不用于持久化或输出地址
|
||||
#[must_use]
|
||||
pub fn with_unreachable_proxy(mut self, endpoint: crate::upstream::ProxyEndpoint) -> Self {
|
||||
self.failure_observation_mut().unreachable_proxy = Some(endpoint);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn unreachable_proxy(&self) -> Option<&crate::upstream::ProxyEndpoint> {
|
||||
self.failure_observation
|
||||
.as_deref()
|
||||
.and_then(|observation| observation.unreachable_proxy.as_ref())
|
||||
}
|
||||
|
||||
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
|
||||
self.failure_observation
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
|
||||
@@ -758,9 +777,21 @@ impl ProviderError {
|
||||
self.pre_delivery_retry = Some(Box::new(PreDeliveryRetry::SameAccountConnectionRetry {
|
||||
transport,
|
||||
}));
|
||||
self.allow_connection_rotation();
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 为瞬时建连失败预留换号机会,不授予重放资格或增加传输重试次数
|
||||
pub fn allow_connection_rotation(&mut self) {
|
||||
self.connection_rotation_allowed = true;
|
||||
}
|
||||
|
||||
/// 供 Core 在传输仍可恢复时为换号预留建连预算
|
||||
#[must_use]
|
||||
pub const fn allows_connection_rotation(&self) -> bool {
|
||||
self.connection_rotation_allowed
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -993,6 +1024,15 @@ impl ProviderError {
|
||||
self.client_visible_upstream_response.as_deref()
|
||||
}
|
||||
|
||||
/// 绑定迁移完成后仅替换客户端恢复投影,保留原始失败的诊断与计费事实
|
||||
pub(crate) fn with_client_recovery_from(mut self, recovery: Self) -> Self {
|
||||
self.client_visible_upstream_error = recovery.client_visible_upstream_error;
|
||||
self.client_visible_upstream_response = recovery.client_visible_upstream_response;
|
||||
self.continuation_failure = recovery.continuation_failure;
|
||||
self.continuation_recovery_disposition = recovery.continuation_recovery_disposition;
|
||||
self
|
||||
}
|
||||
|
||||
/// 取走只供本次重试/提交决策使用的原子客户端事件
|
||||
///
|
||||
/// Core 必须在创建稳定快照、记录中间失败或构造终态前调用本方法,避免原始
|
||||
@@ -1027,6 +1067,7 @@ impl ProviderError {
|
||||
replay_safe: self.replay_safe,
|
||||
retry_prohibited: self.retry_prohibited,
|
||||
same_account_retry_blocked: self.same_account_retry_blocked,
|
||||
connection_rotation_allowed: self.connection_rotation_allowed,
|
||||
pre_delivery_retry: self.pre_delivery_retry.clone(),
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
retry_same_account: self.retry_same_account,
|
||||
@@ -1070,6 +1111,10 @@ impl fmt::Debug for ProviderError {
|
||||
)
|
||||
.field("connection_observation", &self.connection_observation())
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field(
|
||||
"connection_rotation_allowed",
|
||||
&self.connection_rotation_allowed,
|
||||
)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
"credential_recovery_required",
|
||||
|
||||
@@ -101,6 +101,7 @@ impl CompiledSettings {
|
||||
u64::from(settings.openai_session_affinity_ttl_hours) * 3600,
|
||||
))
|
||||
.with_smart_scheduling(settings.smart_scheduling)
|
||||
.with_quota_reset_window_weights(settings.quota_reset_window_weights)
|
||||
.with_queue(ConcurrencyQueuePolicy {
|
||||
max_waiting: settings.max_waiting_per_account,
|
||||
timeout: Duration::from_secs(u64::from(settings.account_queue_timeout_seconds)),
|
||||
|
||||
@@ -30,6 +30,8 @@ pub struct SettingsValues {
|
||||
pub(crate) openai_session_affinity_ttl_hours: u32,
|
||||
pub(crate) responses_max_decompressed_body_bytes: u64,
|
||||
pub(crate) smart_scheduling: crate::account::SmartSchedulingConfig,
|
||||
#[serde(default)]
|
||||
pub(crate) quota_reset_window_weights: crate::account::ResetWindowWeights,
|
||||
pub(crate) rotation_strategy: String,
|
||||
pub(crate) model_mappings: Arc<BTreeMap<String, String>>,
|
||||
pub(crate) min_codex_desktop_version: Option<String>,
|
||||
@@ -111,6 +113,15 @@ impl SettingsValues {
|
||||
&self.request_profiles
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_quota_reset_window_weights(
|
||||
mut self,
|
||||
weights: crate::account::ResetWindowWeights,
|
||||
) -> Self {
|
||||
self.quota_reset_window_weights = weights;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_smart_scheduling(
|
||||
mut self,
|
||||
@@ -201,6 +212,7 @@ impl SettingsValues {
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: crate::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: crate::account::ResetWindowWeights::default(),
|
||||
rotation_strategy: rotation_strategy.into(),
|
||||
model_mappings: Arc::new(model_mappings),
|
||||
min_codex_desktop_version,
|
||||
|
||||
@@ -4,6 +4,32 @@ use std::{fmt, num::NonZeroU32};
|
||||
|
||||
use crate::validation::{IdentifierError, validate_text};
|
||||
|
||||
/// 拨号目标只按主机与端口识别;认证、协议与连接上限不改变 TCP 入口
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProxyEndpoint {
|
||||
host: String,
|
||||
port: u16,
|
||||
}
|
||||
|
||||
impl ProxyEndpoint {
|
||||
pub(crate) fn new(host: &str, port: u16) -> Self {
|
||||
Self {
|
||||
host: host.to_ascii_lowercase(),
|
||||
port,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, host: &str, port: u16) -> bool {
|
||||
self.port == port && host.eq_ignore_ascii_case(&self.host)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProxyEndpoint {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ProxyEndpoint(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// 上游是否可能已经收到业务 payload
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum UpstreamSendState {
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
mod location;
|
||||
mod model_access;
|
||||
mod proxy;
|
||||
mod quota_reset_priority;
|
||||
mod quota_settings;
|
||||
mod scope;
|
||||
mod selection;
|
||||
mod selection_diagnostics;
|
||||
|
||||
@@ -15,6 +15,35 @@ fn proxy_endpoints_support_explicit_schemes_and_redact_credentials() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dial_endpoints_match_host_and_port_without_credentials_or_capacity() {
|
||||
for (source, same, different) in [
|
||||
(
|
||||
"http://user:password@Proxy.Example",
|
||||
"socks5://other:secret@proxy.example:80",
|
||||
"http://proxy.example:81",
|
||||
),
|
||||
(
|
||||
"https://user:password@[::1]",
|
||||
"http://other:secret@[::1]:443",
|
||||
"https://[::2]",
|
||||
),
|
||||
] {
|
||||
let endpoint = OutboundProxy::parse(source).unwrap().dial_endpoint();
|
||||
let same = OutboundProxy::parse(same)
|
||||
.unwrap()
|
||||
.with_max_websocket_connections(3);
|
||||
assert_eq!(endpoint, same.dial_endpoint());
|
||||
assert!(same.matches_endpoint(&endpoint));
|
||||
assert!(
|
||||
!OutboundProxy::parse(different)
|
||||
.unwrap()
|
||||
.matches_endpoint(&endpoint)
|
||||
);
|
||||
assert_eq!(format!("{endpoint:?}"), "ProxyEndpoint(<redacted>)");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_proxy_never_silently_becomes_direct() {
|
||||
for url in [
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//! 验证重置窗口独立排序、相对系数及亲和与可用性边界
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::account::{
|
||||
AccountQuotaResetWindow, AccountSelector, QuotaResetPeriod, ResetWindowWeights,
|
||||
RotationStrategy,
|
||||
};
|
||||
|
||||
use super::{candidate, context};
|
||||
|
||||
#[test]
|
||||
fn selected_windows_rank_strictly_without_mixing_runtime_scores_or_rotating_near_ties() {
|
||||
let mut context = context(RotationStrategy::QuotaResetPriority);
|
||||
let mut early_weekly = candidate("acct_early_weekly", 2, Some(100));
|
||||
early_weekly.signals.failure_rate_basis_points = Some(9_000);
|
||||
early_weekly.signals.first_output_latency_ms = Some(20_000);
|
||||
let mut late_weekly = candidate("acct_late_weekly", 0, Some(9_900));
|
||||
late_weekly.signals.failure_rate_basis_points = Some(0);
|
||||
late_weekly.signals.first_output_latency_ms = Some(10);
|
||||
for (candidate, short, weekly) in [
|
||||
(&mut early_weekly, 14_400, 6 * 86_400),
|
||||
(&mut late_weekly, 3_600, 6 * 86_400 + 43_200),
|
||||
] {
|
||||
candidate.signals.quota_reset_at = Some(context.now + Duration::from_secs(short));
|
||||
candidate.signals.quota_reset_windows = Some(
|
||||
[
|
||||
(QuotaResetPeriod::FiveHour, short),
|
||||
(QuotaResetPeriod::Weekly, weekly),
|
||||
]
|
||||
.map(|(period, seconds)| AccountQuotaResetWindow {
|
||||
period,
|
||||
reset_at: context.now + Duration::from_secs(seconds),
|
||||
})
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let mut candidates = [early_weekly, late_weekly];
|
||||
for (weights, expected) in [
|
||||
([0.0, 0.0, 0.0], "acct_late_weekly"),
|
||||
([1.0, 0.0, 0.0], "acct_late_weekly"),
|
||||
([0.0, 0.1, 0.0], "acct_early_weekly"),
|
||||
([0.0, 1.0, 0.0], "acct_early_weekly"),
|
||||
([0.0, 10.0, 0.0], "acct_early_weekly"),
|
||||
([1.0, 0.1, 0.0], "acct_late_weekly"),
|
||||
([10.0, 1.0, 0.0], "acct_late_weekly"),
|
||||
([0.1, 10.0, 0.0], "acct_early_weekly"),
|
||||
] {
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_quota_reset_window_weights(ResetWindowWeights::new(weights).unwrap());
|
||||
for cursor in 0..6 {
|
||||
context.round_robin_cursor = cursor;
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
expected,
|
||||
"weights {weights:?}, cursor {cursor}"
|
||||
);
|
||||
}
|
||||
}
|
||||
context.preferred_account = Some(candidates[1].account.id().clone());
|
||||
let selected = AccountSelector.select(&candidates, &context).unwrap();
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_late_weekly"
|
||||
);
|
||||
assert_eq!(
|
||||
selected.preferred(),
|
||||
gateway_core::account::PreferredAccountSelection::Hit
|
||||
);
|
||||
|
||||
context.preferred_account = None;
|
||||
candidates[0].signals.in_flight = 3;
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
"acct_late_weekly"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_elapsed_selected_windows_do_not_fall_back_to_disabled_periods() {
|
||||
let mut context = context(RotationStrategy::QuotaResetPriority);
|
||||
context.policy = context
|
||||
.policy
|
||||
.with_quota_reset_window_weights(ResetWindowWeights::new([0.0, 0.0, 1.0]).unwrap());
|
||||
let mut known = candidate("acct_known_monthly", 1, None);
|
||||
known.signals.quota_reset_windows = Some(
|
||||
vec![AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Monthly,
|
||||
reset_at: context.now + Duration::from_secs(30 * 86_400),
|
||||
}]
|
||||
.into(),
|
||||
);
|
||||
for reset_at in [
|
||||
None,
|
||||
Some(context.now),
|
||||
Some(context.now - Duration::from_secs(1)),
|
||||
] {
|
||||
let mut missing = candidate("acct_earlier_other_period", 0, None);
|
||||
missing.signals.quota_reset_at = Some(context.now + Duration::from_secs(1));
|
||||
missing.signals.quota_reset_windows = Some(
|
||||
reset_at
|
||||
.into_iter()
|
||||
.map(|reset_at| AccountQuotaResetWindow {
|
||||
period: QuotaResetPeriod::Monthly,
|
||||
reset_at,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.into(),
|
||||
);
|
||||
let candidates = [missing, known.clone()];
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.unwrap()
|
||||
.candidate()
|
||||
.account
|
||||
.id()
|
||||
.as_str(),
|
||||
"acct_known_monthly"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//! 额度桶配置校验与上限、预热的独立行为
|
||||
|
||||
use gateway_core::account::AccountQuotaSettings;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn warmup_and_limit_can_be_configured_independently() {
|
||||
let settings: AccountQuotaSettings = serde_json::from_value(json!({
|
||||
"codex:18000s": {"warmup": {"model": "test-model"}},
|
||||
"codex:604800s": {"limitPercent": 95}
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(settings.has_warmup());
|
||||
assert!(!settings.reached("codex:18000s", Some(100.0)));
|
||||
assert!(!settings.reached("codex:604800s", Some(94.9)));
|
||||
assert!(settings.reached("codex:604800s", Some(95.0)));
|
||||
assert!(!settings.reached("codex:604800s", None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_bucket_settings_are_rejected() {
|
||||
for invalid in [
|
||||
json!(95),
|
||||
json!({"limitPercent": 0}),
|
||||
json!({"limitPercent": 101}),
|
||||
json!({"warmup": {}}),
|
||||
json!({"warmup": {"model": ""}}),
|
||||
json!({"warmup": {"model": " model"}}),
|
||||
json!({"warmup": {"model": "model\nname"}}),
|
||||
json!({"warmup": {"model": "x".repeat(129)}}),
|
||||
json!({"warmup": {"model": "model", "unexpected": true}}),
|
||||
] {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountQuotaSettings>(json!({"codex:18000s": invalid}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1228,6 +1228,7 @@ fn plan_with_protection(
|
||||
None,
|
||||
)
|
||||
.with_smart_scheduling(account_selection_policy.smart_scheduling())
|
||||
.with_quota_reset_window_weights(account_selection_policy.quota_reset_window_weights())
|
||||
.with_max_account_rotations(account_selection_policy.max_account_rotations())
|
||||
.with_openai_account_affinity(account_selection_policy.openai_account_affinity())
|
||||
.with_openai_session_affinity_ttl_hours(
|
||||
@@ -2490,6 +2491,124 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
for (seconds, replacement, rotations, expected_attempts) in [
|
||||
(309, true, 1, 2),
|
||||
(309, false, 1, 2),
|
||||
(240, true, 1, 1),
|
||||
(309, true, 0, 1),
|
||||
] {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
unreachable!()
|
||||
};
|
||||
let operation =
|
||||
Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).unwrap(),
|
||||
));
|
||||
let route_plan = plan_with_protection(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::MIN)
|
||||
.with_max_account_rotations(rotations),
|
||||
Default::default(),
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
240,
|
||||
);
|
||||
let original_body = Bytes::from_static(br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"309","retry-after-ms":"309000"}}}"#);
|
||||
let original = transient_rejection()
|
||||
.with_retry_after(Duration::from_secs(seconds))
|
||||
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
|
||||
String::from_utf8(original_body.to_vec()).unwrap(),
|
||||
))
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
);
|
||||
let recovery = if replacement {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Resend full history.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
))
|
||||
} else {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(original)],
|
||||
},
|
||||
Script::Error(recovery),
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("client-previous"),
|
||||
PreviousResponseId::new("upstream-previous"),
|
||||
ClientApiKeyId::new("key_client_1").unwrap(),
|
||||
ProviderKind::new("openai").unwrap(),
|
||||
ProviderAccountId::new("acct_one").unwrap(),
|
||||
)
|
||||
.with_scope(gateway_core::engine::continuation::NativeContinuationScope::ConnectionLocal);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) =
|
||||
block_on(async { futures::poll!(&mut collection) })
|
||||
else {
|
||||
panic!("native capacity recovery must not wait on the old account");
|
||||
};
|
||||
drop(collection);
|
||||
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(seconds)));
|
||||
assert_eq!(
|
||||
error.raw_upstream_error().unwrap().as_str().as_bytes(),
|
||||
original_body.as_ref()
|
||||
);
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_error()
|
||||
.and_then(|detail| detail.code()),
|
||||
(replacement && expected_attempts == 2).then_some("previous_response_not_found")
|
||||
);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), expected_attempts);
|
||||
if contexts.len() == 2 {
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::RotateForClientReplay
|
||||
);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_one").unwrap())
|
||||
);
|
||||
assert!(contexts[1].required_account().is_none());
|
||||
}
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_client_replay_required_is_terminal() {
|
||||
for (kind, send_state) in [
|
||||
|
||||
@@ -877,11 +877,15 @@ fn compiler_accepts_unlimited_default_account_concurrency() {
|
||||
|
||||
#[test]
|
||||
fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
use gateway_core::account::SmartSchedulingConfig;
|
||||
use gateway_core::account::{ResetWindowWeights, SmartSchedulingConfig};
|
||||
let configs = [
|
||||
SmartSchedulingConfig::default(),
|
||||
SmartSchedulingConfig::new([0.0, 2.0, 1.0, 0.5, 1.2, 2.3]).unwrap(),
|
||||
];
|
||||
let windows = [
|
||||
ResetWindowWeights::default(),
|
||||
ResetWindowWeights::new([0.0, 1.5, 0.0]).unwrap(),
|
||||
];
|
||||
let mut plans = Vec::new();
|
||||
for (index, config) in configs.into_iter().enumerate() {
|
||||
let facts = SnapshotFacts::new(
|
||||
@@ -889,6 +893,7 @@ fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
revision(index as u64 + 1),
|
||||
SettingsValues::new(3, "smart", BTreeMap::new(), None, None)
|
||||
.with_smart_scheduling(config)
|
||||
.with_quota_reset_window_weights(windows[index])
|
||||
.with_openai_guardian_reserved_concurrency(index as u32),
|
||||
vec![],
|
||||
vec![],
|
||||
@@ -921,6 +926,10 @@ fn compiled_plans_keep_their_scheduling_config_after_a_new_snapshot_is_built() {
|
||||
index as u32
|
||||
);
|
||||
assert_eq!(plan.account_selection_policy().smart_scheduling(), config);
|
||||
assert_eq!(
|
||||
plan.account_selection_policy().quota_reset_window_weights(),
|
||||
windows[index]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ gateway-plugin-sdk = { path = "../sdk", features = ["io"] }
|
||||
gateway-protocol = { path = "../../gateway-protocol" }
|
||||
hex = "0.4.3"
|
||||
image = { version = "0.25.8", default-features = false, features = ["bmp", "gif", "ico", "jpeg", "png", "webp"] }
|
||||
jsonschema = { version = "0.60.0", default-features = false }
|
||||
jsonschema = { version = "0.61.2", default-features = false }
|
||||
roxmltree = "0.21.1"
|
||||
semver = "1.0.27"
|
||||
secrecy = "0.10.3"
|
||||
|
||||
@@ -33,7 +33,7 @@ fn account() -> AccountRecord {
|
||||
let at = "2026-01-01T00:00:00Z".parse().unwrap();
|
||||
AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
@@ -154,6 +154,7 @@ pub struct RuntimeSettings {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
@@ -194,6 +195,7 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: SmartSchedulingConfig,
|
||||
pub quota_reset_window_weights: ResetWindowWeights,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
@@ -405,6 +407,7 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
|
||||
openai_session_affinity_ttl_hours: settings.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
quota_reset_window_weights: settings.quota_reset_window_weights,
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: settings.usage_retention_days,
|
||||
|
||||
@@ -5,6 +5,7 @@ mod content;
|
||||
mod error;
|
||||
mod request;
|
||||
mod response;
|
||||
mod session;
|
||||
mod stream;
|
||||
mod thinking;
|
||||
mod tools;
|
||||
@@ -20,6 +21,7 @@ pub use request::{
|
||||
pub use response::{
|
||||
messages_to_responses_event, messages_to_responses_response, responses_to_messages_response,
|
||||
};
|
||||
pub use session::claude_session_id;
|
||||
pub use stream::{
|
||||
MessagesAccumulator, MessagesToResponsesStream, ResponsesToMessagesStream,
|
||||
message_to_stream_events,
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
//! Claude Code 明确会话身份的请求头与 metadata 解析
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// 优先使用 Claude Code 会话头,再读取新旧 metadata.user_id 中的会话标识
|
||||
///
|
||||
/// 普通用户标识与对话内容不作为会话身份
|
||||
#[must_use]
|
||||
pub fn claude_session_id(body: &Map<String, Value>, header: Option<&str>) -> Option<String> {
|
||||
if let Some(session) = header.and_then(valid_session_id) {
|
||||
return Some(session.to_owned());
|
||||
}
|
||||
let user_id = body.get("metadata")?.get("user_id")?.as_str()?.trim();
|
||||
if user_id.starts_with('{') {
|
||||
let metadata: Value = serde_json::from_str(user_id).ok()?;
|
||||
return metadata
|
||||
.get("session_id")?
|
||||
.as_str()
|
||||
.and_then(valid_session_id)
|
||||
.map(str::to_owned);
|
||||
}
|
||||
let (_, session) = user_id.rsplit_once("_session_")?;
|
||||
let session = valid_session_id(session)?;
|
||||
session
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() || byte == b'-')
|
||||
.then(|| session.to_owned())
|
||||
}
|
||||
|
||||
fn valid_session_id(value: &str) -> Option<&str> {
|
||||
if value.chars().any(char::is_control) {
|
||||
return None;
|
||||
}
|
||||
let value = value.trim();
|
||||
(!value.is_empty() && value.len() <= 256).then_some(value)
|
||||
}
|
||||
@@ -3,5 +3,6 @@
|
||||
mod capabilities;
|
||||
mod request;
|
||||
mod response;
|
||||
mod session;
|
||||
mod stream;
|
||||
mod usage;
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
//! Claude Code 会话来源优先级、格式兼容与无效身份边界
|
||||
|
||||
use gateway_protocol::anthropic::claude_session_id;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn explicit_claude_sessions_follow_header_then_metadata() {
|
||||
let session = "12345678-1234-1234-1234-123456789abc";
|
||||
for (header, user_id, expected) in [
|
||||
(
|
||||
Some(" header-session "),
|
||||
json!("{\"session_id\":\"body-session\"}"),
|
||||
Some("header-session"),
|
||||
),
|
||||
(
|
||||
None,
|
||||
json!("{\"device_id\":\"device\",\"session_id\":\"body-session\"}"),
|
||||
Some("body-session"),
|
||||
),
|
||||
(
|
||||
None,
|
||||
json!(format!("user_device_account__session_{session}")),
|
||||
Some(session),
|
||||
),
|
||||
(
|
||||
Some(""),
|
||||
json!(format!("user_{}_session_{session}", "x".repeat(300))),
|
||||
Some(session),
|
||||
),
|
||||
(None, json!("ordinary-user-id"), None),
|
||||
(None, json!("{\"device_id\":\"device\"}"), None),
|
||||
(None, json!("{invalid}"), None),
|
||||
(None, json!({"session_id":"not-a-string"}), None),
|
||||
(None, json!("user_device_session_"), None),
|
||||
(None, json!("user_device_session_not-a-uuid"), None),
|
||||
] {
|
||||
let body =
|
||||
json!({"metadata":{"user_id":user_id},"messages":[{"role":"user","content":"hello"}]});
|
||||
assert_eq!(
|
||||
claude_session_id(body.as_object().unwrap(), header).as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_session_signals_do_not_bind_users_or_message_content() {
|
||||
for invalid in [String::new(), "x".repeat(257), "bad\nidentity".to_owned()] {
|
||||
let body = json!({"metadata":{"user_id":json!({"session_id":invalid}).to_string()}});
|
||||
assert_eq!(
|
||||
claude_session_id(body.as_object().unwrap(), Some(&invalid)),
|
||||
None
|
||||
);
|
||||
let fallback = json!({"metadata":{"user_id":"{\"session_id\":\"valid-fallback\"}"}});
|
||||
assert_eq!(
|
||||
claude_session_id(fallback.as_object().unwrap(), Some(&invalid)).as_deref(),
|
||||
Some("valid-fallback")
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
claude_session_id(
|
||||
json!({"messages":[{"role":"user","content":"same opening"}]})
|
||||
.as_object()
|
||||
.unwrap(),
|
||||
None
|
||||
),
|
||||
None
|
||||
);
|
||||
}
|
||||
@@ -114,6 +114,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
"responses_max_decompressed_body_bytes".to_owned(),
|
||||
"rotation_strategy".to_owned(),
|
||||
"smart_scheduling_json".to_owned(),
|
||||
"quota_reset_window_weights_json".to_owned(),
|
||||
"min_codex_desktop_version".to_owned(),
|
||||
"min_codex_cli_version".to_owned(),
|
||||
"retention".to_owned(),
|
||||
|
||||
@@ -242,7 +242,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
account.upstream_user_id, account.upstream_account_id, account.plan_type,
|
||||
account.authentication_kind, account.credential_revision, account.outbound_proxy_id, account.outbound_proxy_url,
|
||||
account.has_refresh_token, account.access_token_expires_at,
|
||||
account.next_refresh_at, account.enabled, account.quota_limits, account.quota_paused, account.auto_use_quota_credits, account.concurrency_limit,
|
||||
account.next_refresh_at, account.enabled, account.quota_settings, account.quota_paused, account.auto_use_quota_credits, account.concurrency_limit,
|
||||
account.quota_observed_at, account.weight, account.model_access_json, account.credential_state, account.quota_access_state,
|
||||
account.quota_evidence, account.quota_access_observed_at,
|
||||
account.quota_reset_at, account.last_error_reason,
|
||||
|
||||
@@ -665,7 +665,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
|
||||
account_ids: vec![command.account_id.clone()],
|
||||
pauses: Default::default(),
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
name: command.name,
|
||||
notes: command.notes,
|
||||
@@ -751,7 +751,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
};
|
||||
let mut changed_fields = Vec::new();
|
||||
for (changed, field) in [
|
||||
(command.quota_limits.is_some(), "quota_limits"),
|
||||
(command.quota_settings.is_some(), "quota_settings"),
|
||||
(
|
||||
command.auto_use_quota_credits.is_some(),
|
||||
"auto_use_quota_credits",
|
||||
@@ -776,7 +776,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
|
||||
account_ids: command.account_ids,
|
||||
pauses,
|
||||
quota_limits: command.quota_limits,
|
||||
quota_settings: command.quota_settings,
|
||||
auto_use_quota_credits: command.auto_use_quota_credits,
|
||||
name: None,
|
||||
notes: None,
|
||||
|
||||
@@ -108,7 +108,7 @@ pub(crate) async fn load_admin_account_page(
|
||||
select coalesce(p.enabled, true) as outbound_proxy_enabled, coalesce(p.max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, p.auto_location, p.detected_location_json, p.location_country, p.location_region, p.location_city, p.location_timezone, a.outbound_proxy_id, a.outbound_proxy_url, a.id, a.provider_kind, a.name, a.notes, a.email, a.upstream_user_id,
|
||||
a.upstream_account_id, a.plan_type, a.authentication_kind,
|
||||
a.credential_revision, a.has_refresh_token, a.access_token_expires_at,
|
||||
a.next_refresh_at, a.enabled, a.quota_limits, a.quota_paused, a.auto_use_quota_credits, a.concurrency_limit, a.weight, a.model_access_json,
|
||||
a.next_refresh_at, a.enabled, a.quota_settings, a.quota_paused, a.auto_use_quota_credits, a.concurrency_limit, a.weight, a.model_access_json,
|
||||
a.credential_state, a.quota_access_state, a.quota_evidence,
|
||||
a.quota_observed_at, a.quota_access_observed_at, a.quota_reset_at, a.last_error_reason,
|
||||
a.last_error_message, a.credential_observed_at, a.created_at, a.updated_at,
|
||||
|
||||
@@ -14,6 +14,24 @@ fn loaded_credential_from_record(
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
async fn load_quota_warmup_state(
|
||||
&self,
|
||||
account: &CoreProviderAccount,
|
||||
) -> Result<Option<gateway_core::account::QuotaWarmupState>, CoreStoreError> {
|
||||
self.load_warmup_state(account)
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn compare_and_swap_quota_warmup_state(
|
||||
&self,
|
||||
update: gateway_core::account::QuotaWarmupStateUpdate<'_>,
|
||||
) -> Result<bool, CoreStoreError> {
|
||||
self.write_warmup_state(update)
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn load_account_catalogs(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
@@ -246,7 +264,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
set name = case when $14 then name else $3 end,
|
||||
email = case when $14 then email else $4 end,
|
||||
plan_type = case when $14 then plan_type else $5 end,
|
||||
provider_quota_json = case when $14 then provider_quota_json else null end, quota_paused = case when $14 then quota_paused else false end,
|
||||
quota_warmup_state = case when $14 then quota_warmup_state else '{}'::jsonb end, quota_warmup_revision = quota_warmup_revision + 1, provider_quota_json = case when $14 then provider_quota_json else null end, quota_paused = case when $14 then quota_paused else false end,
|
||||
quota_observed_at = case when $14 then quota_observed_at else null end,
|
||||
provider_credentials_json = $6,
|
||||
credential_revision = credential_revision + 1,
|
||||
|
||||
@@ -11,7 +11,7 @@ pub(crate) fn admin_account_record(
|
||||
) -> AdminStoreResult<AccountRecord> {
|
||||
Ok(AccountRecord {
|
||||
quota_paused: summary.quota_paused,
|
||||
quota_limits: summary.quota_limits,
|
||||
quota_settings: summary.quota_settings,
|
||||
auto_use_quota_credits: summary.auto_use_quota_credits,
|
||||
outbound_proxy_id: summary.outbound_proxy_id,
|
||||
outbound_proxy_enabled: summary.outbound_proxy_enabled,
|
||||
|
||||
@@ -61,6 +61,7 @@ mod mapping;
|
||||
mod model_catalogs;
|
||||
mod quota;
|
||||
mod quota_forecast;
|
||||
mod quota_warmup;
|
||||
mod repository;
|
||||
mod reset_credit_automation;
|
||||
mod rows;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
//! 额度桶预热进度的版本读取和原子领取
|
||||
|
||||
use super::*;
|
||||
use gateway_core::account::{QuotaWarmupState, QuotaWarmupStateUpdate};
|
||||
|
||||
impl PgProviderAccountRepository {
|
||||
pub(super) async fn load_warmup_state(
|
||||
&self,
|
||||
account: &CoreProviderAccount,
|
||||
) -> StoreResult<Option<QuotaWarmupState>> {
|
||||
let row = sqlx::query(
|
||||
"select quota_warmup_revision, quota_warmup_state from provider_accounts
|
||||
where id = $1 and credential_revision = $2",
|
||||
)
|
||||
.bind(account.id().as_str())
|
||||
.bind(to_i64(account.revision().get())?)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("load quota warmup state", error))?;
|
||||
row.map(|row| {
|
||||
let version: i64 = row
|
||||
.try_get("quota_warmup_revision")
|
||||
.map_err(|error| postgres_unavailable("read quota warmup revision", error))?;
|
||||
let data: serde_json::Value = row
|
||||
.try_get("quota_warmup_state")
|
||||
.map_err(|error| postgres_unavailable("read quota warmup state", error))?;
|
||||
Ok(QuotaWarmupState {
|
||||
version: u64::try_from(version)
|
||||
.map_err(|_| invalid("invalid quota warmup revision"))?,
|
||||
data: data
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| invalid("invalid quota warmup state"))?,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(super) async fn write_warmup_state(
|
||||
&self,
|
||||
update: QuotaWarmupStateUpdate<'_>,
|
||||
) -> StoreResult<bool> {
|
||||
let data = JsonObject::try_from_value(
|
||||
"quota_warmup_state",
|
||||
serde_json::Value::Object(update.data),
|
||||
QUOTA_MAX_BYTES,
|
||||
)?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts set quota_warmup_state = $4,
|
||||
quota_warmup_revision = quota_warmup_revision + 1
|
||||
where id = $1 and credential_revision = $2 and quota_warmup_revision = $3
|
||||
and quota_settings = $5 and enabled
|
||||
and quota_observed_at is not distinct from $6
|
||||
and credential_state = $7 and quota_access_state = $8 and quota_paused = $9 and model_access_json = $10",
|
||||
)
|
||||
.bind(update.account.id().as_str())
|
||||
.bind(to_i64(update.account.revision().get())?)
|
||||
.bind(to_i64(update.expected_version)?)
|
||||
.bind(serde_json::Value::Object(data.fields().clone()))
|
||||
.bind(
|
||||
serde_json::to_value(update.account.quota_settings())
|
||||
.map_err(|_| invalid("invalid quota settings"))?,
|
||||
)
|
||||
.bind(update.expected_quota_observed_at.map(DateTime::<Utc>::from))
|
||||
.bind(update.account.credential_state().as_str())
|
||||
.bind(update.account.quota().access().as_str())
|
||||
.bind(update.account.quota_paused())
|
||||
.bind(serde_json::to_value(update.account.model_access()).map_err(|_| invalid("invalid model access"))?)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("write quota warmup state", error))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
}
|
||||
@@ -109,7 +109,7 @@ impl PgProviderAccountRepository {
|
||||
let mut query = sqlx::QueryBuilder::<Postgres>::new(
|
||||
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
credential_observed_at, quota_access_state, quota_evidence,
|
||||
quota_access_observed_at, quota_reset_at,
|
||||
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
|
||||
@@ -207,7 +207,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let rows = sqlx::query(
|
||||
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
credential_observed_at, quota_access_state, quota_evidence,
|
||||
quota_access_observed_at, quota_reset_at,
|
||||
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
|
||||
@@ -327,7 +327,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
}
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update provider_accounts
|
||||
set provider_quota_json = case when $7 then provider_quota_json else null end, quota_paused = case when $7 then quota_paused else false end,
|
||||
set quota_warmup_state = case when $7 then quota_warmup_state else '{}'::jsonb end, quota_warmup_revision = provider_accounts.quota_warmup_revision + 1, provider_quota_json = case when $7 then provider_quota_json else null end, quota_paused = case when $7 then quota_paused else false end,
|
||||
quota_observed_at = case when $7 then quota_observed_at else null end,
|
||||
provider_credentials_json = $3,
|
||||
credential_revision = credential_revision + 1,
|
||||
@@ -531,17 +531,16 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
})?;
|
||||
let result = async {
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
if let Some(limits) = &command.quota_limits {
|
||||
if command.quota_settings.is_some() {
|
||||
for id in &command.account_ids {
|
||||
let pause = command.pauses.get(id);
|
||||
if !limits.is_empty() && pause.is_none() {
|
||||
return Err(invalid("缺少账号额度暂停结论"));
|
||||
}
|
||||
let updated = sqlx::query("update provider_accounts set quota_limits = $2, quota_paused = $3
|
||||
where id = $1 and ($4::bigint is null or (credential_revision = $4 and quota_observed_at is not distinct from $5))")
|
||||
.bind(id).bind(sqlx::types::Json(limits)).bind(pause.is_some_and(|p| p.paused))
|
||||
.bind(pause.map(|p| to_i64(p.revision.get())).transpose()?).bind(pause.and_then(|p| p.observed_at))
|
||||
.execute(&mut *transaction).await.map_err(|source| postgres_unavailable("set account quota limits", source))?;
|
||||
let pause = command.pauses.get(id).ok_or_else(|| invalid("缺少额度桶设置结论"))?;
|
||||
let updated = sqlx::query("update provider_accounts set quota_settings = $2, quota_paused = $3
|
||||
where id = $1 and credential_revision = $4 and quota_settings = $6
|
||||
and ($7 or quota_observed_at is not distinct from $5)")
|
||||
.bind(id).bind(sqlx::types::Json(&pause.settings)).bind(pause.paused)
|
||||
.bind(to_i64(pause.revision.get())?).bind(pause.observed_at)
|
||||
.bind(sqlx::types::Json(&pause.previous_settings)).bind(pause.settings.is_empty())
|
||||
.execute(&mut *transaction).await.map_err(|source| postgres_unavailable("set account quota settings", source))?;
|
||||
if updated.rows_affected() == 0 {
|
||||
return Err(StoreError::Conflict { source: None, entity: ENTITY, id: id.clone(), kind: ConflictKind::StaleRevision });
|
||||
}
|
||||
@@ -613,7 +612,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
when access_token_expires_at <= now() then null
|
||||
else access_token_expires_at
|
||||
end,
|
||||
provider_quota_json = null, quota_paused = false,
|
||||
provider_quota_json = null, quota_paused = false, quota_warmup_state = '{}'::jsonb, quota_warmup_revision = provider_accounts.quota_warmup_revision + 1,
|
||||
quota_observed_at = null,
|
||||
quota_access_state = 'allowed',
|
||||
quota_evidence = null,
|
||||
@@ -790,7 +789,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
next_refresh_at = excluded.next_refresh_at,
|
||||
enabled = excluded.enabled,
|
||||
credential_state = excluded.credential_state,
|
||||
provider_quota_json = null, quota_paused = false,
|
||||
provider_quota_json = null, quota_paused = false, quota_warmup_state = '{}'::jsonb, quota_warmup_revision = provider_accounts.quota_warmup_revision + 1,
|
||||
quota_access_state = 'unknown',
|
||||
quota_evidence = null,
|
||||
quota_access_observed_at = null,
|
||||
@@ -866,7 +865,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
set name = case when $14 or $16 then name else $4 end,
|
||||
email = case when $14 then email else $5 end,
|
||||
plan_type = case when $14 then plan_type else $6 end,
|
||||
provider_quota_json = case when $14 then provider_quota_json else null end, quota_paused = case when $14 then quota_paused else false end,
|
||||
quota_warmup_state = case when $14 then quota_warmup_state else '{}'::jsonb end, quota_warmup_revision = provider_accounts.quota_warmup_revision + 1, provider_quota_json = case when $14 then provider_quota_json else null end, quota_paused = case when $14 then quota_paused else false end,
|
||||
quota_observed_at = case when $14 then quota_observed_at else null end,
|
||||
quota_access_state = case when $11 then 'unknown' else quota_access_state end,
|
||||
quota_evidence = case when $11 then null else quota_evidence end,
|
||||
@@ -1256,7 +1255,7 @@ impl PgProviderAccountRepository {
|
||||
let rows = sqlx::query(
|
||||
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
credential_observed_at, quota_access_state, quota_evidence,
|
||||
quota_access_observed_at, quota_reset_at,
|
||||
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
|
||||
@@ -1354,7 +1353,7 @@ where
|
||||
updated_at = greatest(now(), updated_at, $4, $7)
|
||||
where id = $1 and credential_revision = $2
|
||||
and (quota_observed_at is null or quota_observed_at <= $4)
|
||||
and ($11::jsonb is null or (quota_limits = $11 and quota_observed_at is distinct from $4))
|
||||
and ($11::jsonb is null or (quota_settings = $11 and quota_observed_at is distinct from $4))
|
||||
and (not $12 or quota_observed_at is not distinct from $13)",
|
||||
)
|
||||
.bind(account_id)
|
||||
|
||||
@@ -52,7 +52,7 @@ pub(crate) fn parse_error_reason(value: Option<String>) -> StoreResult<Option<Ac
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountSummary {
|
||||
pub quota_paused: bool,
|
||||
pub quota_limits: gateway_core::account::AccountQuotaLimits,
|
||||
pub quota_settings: gateway_core::account::AccountQuotaSettings,
|
||||
pub auto_use_quota_credits: bool,
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
pub outbound_proxy_enabled: bool,
|
||||
@@ -253,7 +253,7 @@ impl fmt::Debug for RotateProviderAccount {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BatchUpdateProviderAccountsAdmin {
|
||||
pub pauses: gateway_admin::model::accounts::PreparedQuotaPauses,
|
||||
pub quota_limits: Option<gateway_core::account::AccountQuotaLimits>,
|
||||
pub quota_settings: Option<gateway_core::account::AccountQuotaSettings>,
|
||||
pub auto_use_quota_credits: Option<bool>,
|
||||
pub outbound_proxy: Option<gateway_admin::model::proxies::AccountProxySelection>,
|
||||
pub account_ids: Vec<String>,
|
||||
@@ -328,7 +328,7 @@ impl ProviderAccountStateUpdate {
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
|
||||
last_error_reason, last_error_message,
|
||||
credential_observed_at, quota_observed_at, created_at, updated_at
|
||||
@@ -339,7 +339,7 @@ pub(crate) const ACCOUNT_SELECT: &str = "select coalesce(proxy_enabled, true) as
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
|
||||
last_error_reason, last_error_message,
|
||||
credential_observed_at, quota_observed_at, created_at, updated_at
|
||||
@@ -351,7 +351,7 @@ pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select coalesce(proxy_enabled, t
|
||||
|
||||
pub(crate) const REFRESH_CANDIDATES_SELECT: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_limits, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, quota_settings, quota_paused, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
|
||||
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
|
||||
last_error_reason, last_error_message,
|
||||
credential_observed_at, quota_observed_at, created_at, updated_at
|
||||
@@ -421,7 +421,7 @@ pub(crate) fn core_account_from_summary(
|
||||
)
|
||||
.with_scheduling(summary.concurrency_limit, summary.weight)
|
||||
.with_model_access(summary.model_access)
|
||||
.with_quota_limits(summary.quota_limits, summary.quota_paused)
|
||||
.with_quota_settings(summary.quota_settings, summary.quota_paused)
|
||||
.with_auto_use_quota_credits(summary.auto_use_quota_credits)
|
||||
.with_outbound_proxy(
|
||||
summary
|
||||
@@ -479,9 +479,9 @@ pub(crate) fn account_summary_from_row(
|
||||
.ok_or_else(|| invalid("invalid weight"))?;
|
||||
Ok(ProviderAccountSummary {
|
||||
quota_paused: status.quota_paused,
|
||||
quota_limits: get::<sqlx::types::Json<gateway_core::account::AccountQuotaLimits>>(
|
||||
quota_settings: get::<sqlx::types::Json<gateway_core::account::AccountQuotaSettings>>(
|
||||
&row,
|
||||
"quota_limits",
|
||||
"quota_settings",
|
||||
)?
|
||||
.0,
|
||||
auto_use_quota_credits: get(&row, "auto_use_quota_credits")?,
|
||||
|
||||
@@ -153,7 +153,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
|
||||
let row = sqlx::query_as::<_, RuntimeSettingsRow>(
|
||||
"select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, codex_privacy_policy_json,
|
||||
refresh_concurrency, max_concurrent_per_account,
|
||||
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
rotation_strategy, smart_scheduling_json, quota_reset_window_weights_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
@@ -333,7 +333,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
let row = sqlx::query_as::<_, RuntimeSettingsRow>(
|
||||
"select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, codex_privacy_policy_json,
|
||||
refresh_concurrency, max_concurrent_per_account,
|
||||
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
rotation_strategy, smart_scheduling_json, quota_reset_window_weights_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
@@ -419,6 +419,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
account_queue_timeout_seconds = $32,
|
||||
switch_account_on_queue_failure = $33,
|
||||
reset_credit_on_low_weekly_quota = $34,
|
||||
quota_reset_window_weights_json = $35,
|
||||
updated_at = now()
|
||||
where id = 1
|
||||
returning config_revision",
|
||||
@@ -465,6 +466,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
.bind(i64::from(update.values.account_queue_timeout_seconds))
|
||||
.bind(update.values.switch_account_on_queue_failure)
|
||||
.bind(update.values.reset_credit_on_low_weekly_quota)
|
||||
.bind(sqlx::types::Json(update.values.quota_reset_window_weights))
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("update runtime settings in transaction", source))?
|
||||
@@ -538,6 +540,7 @@ struct RuntimeSettingsRow {
|
||||
refresh_concurrency: i64,
|
||||
max_concurrent_per_account: i64,
|
||||
smart_scheduling_json: sqlx::types::Json<gateway_core::account::SmartSchedulingConfig>,
|
||||
quota_reset_window_weights_json: sqlx::types::Json<gateway_core::account::ResetWindowWeights>,
|
||||
rotation_strategy: String,
|
||||
codex_privacy_policy_json:
|
||||
sqlx::types::Json<gateway_core::settings::privacy::CodexPrivacyPolicy>,
|
||||
@@ -600,6 +603,7 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
|
||||
refresh_concurrency: to_u32(row.refresh_concurrency)?,
|
||||
max_concurrent_per_account: to_u32(row.max_concurrent_per_account)?,
|
||||
smart_scheduling: row.smart_scheduling_json.0,
|
||||
quota_reset_window_weights: row.quota_reset_window_weights_json.0,
|
||||
codex_privacy_policy: row.codex_privacy_policy_json.0,
|
||||
request_location_enabled: row.request_location_enabled,
|
||||
request_location: row
|
||||
|
||||
@@ -221,6 +221,7 @@ struct SnapshotSettingsRow {
|
||||
account_error_protection_json: sqlx::types::Json<gateway_core::account::AccountErrorProtection>,
|
||||
max_concurrent_per_account: i64,
|
||||
smart_scheduling_json: sqlx::types::Json<gateway_core::account::SmartSchedulingConfig>,
|
||||
quota_reset_window_weights_json: sqlx::types::Json<gateway_core::account::ResetWindowWeights>,
|
||||
rotation_strategy: String,
|
||||
model_mappings_json: sqlx::types::Json<BTreeMap<String, String>>,
|
||||
min_codex_desktop_version: Option<String>,
|
||||
@@ -248,7 +249,7 @@ async fn load_settings(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> StoreResult<(Revision, SettingsValues)> {
|
||||
let row = sqlx::query_as::<_, SnapshotSettingsRow>(
|
||||
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
|
||||
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, quota_reset_window_weights_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
@@ -281,6 +282,7 @@ async fn load_settings(
|
||||
.with_max_retry_after_seconds(to_u32(row.max_retry_after_seconds)?)
|
||||
.with_openai_session_affinity_ttl_hours(to_u32(row.openai_session_affinity_ttl_hours)?)
|
||||
.with_smart_scheduling(row.smart_scheduling_json.0)
|
||||
.with_quota_reset_window_weights(row.quota_reset_window_weights_json.0)
|
||||
.with_request_profiles(decode_request_profiles(
|
||||
row.provider_request_profiles_json.0,
|
||||
)?)
|
||||
|
||||
@@ -1575,7 +1575,7 @@ async fn account_notes_round_trip_and_survive_import_and_scheduling_updates() {
|
||||
store
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec!["acct_notes".to_owned()],
|
||||
enabled: Some(false),
|
||||
@@ -1817,7 +1817,7 @@ async fn account_enable_preserves_facts_and_explicit_recovery_clears_them() {
|
||||
let enabled = store
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec!["acct_recovery".to_owned()],
|
||||
enabled: Some(true),
|
||||
@@ -1944,7 +1944,7 @@ async fn terminal_batch_update_replaces_state_and_groups_once_or_rolls_back_ever
|
||||
let result = store
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
@@ -1984,7 +1984,7 @@ async fn terminal_batch_update_replaces_state_and_groups_once_or_rolls_back_ever
|
||||
store
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
@@ -2727,7 +2727,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
let revision = repository
|
||||
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
|
||||
pauses: Default::default(),
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
name: None,
|
||||
notes: None,
|
||||
@@ -3766,7 +3766,7 @@ async fn model_access_only_batch_update_preserves_other_settings_and_survives_re
|
||||
.expect("policy");
|
||||
let store = admin_account_store(&database.pool);
|
||||
let command = BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec![input.id.clone()],
|
||||
enabled: None,
|
||||
@@ -3912,7 +3912,7 @@ async fn disabling_quota_credits_revokes_credit_only_access_even_with_a_late_obs
|
||||
let set_policy = async |enabled| {
|
||||
let command = BatchUpdateAccounts {
|
||||
account_ids: vec![input.id.clone()],
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: Some(enabled),
|
||||
enabled: None,
|
||||
concurrency_limit: None,
|
||||
|
||||
@@ -147,14 +147,15 @@ async fn quota_merge_can_only_replace_the_snapshot_it_read() {
|
||||
.insert_provider_account(account(id.as_str(), "fixture"))
|
||||
.await
|
||||
.unwrap();
|
||||
let limits = json!({"codex:18000s": 95});
|
||||
sqlx::query("update provider_accounts set quota_limits = $2 where id = $1")
|
||||
let limits = json!({"codex:18000s": {"limitPercent": 95, "warmup": null}});
|
||||
sqlx::query("update provider_accounts set quota_settings = $2 where id = $1")
|
||||
.bind(id.as_str())
|
||||
.bind(&limits)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let limits: gateway_core::account::AccountQuotaLimits = serde_json::from_value(limits).unwrap();
|
||||
let limits: gateway_core::account::AccountQuotaSettings =
|
||||
serde_json::from_value(limits).unwrap();
|
||||
let start = SystemTime::UNIX_EPOCH + Duration::from_secs(1_800_000_000);
|
||||
let later = start + Duration::from_secs(1);
|
||||
let observed = |base, observed_at, paused| QuotaObservation {
|
||||
@@ -417,3 +418,68 @@ async fn managed_proxy_changes_clear_snapshots_and_reject_old_connections() {
|
||||
assert!(repository.get_quotas(&[id]).await.unwrap().is_empty());
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_warmup_claim_is_persistent_and_fenced_by_configuration_usage_and_account_state() {
|
||||
use gateway_core::account::QuotaWarmupStateUpdate;
|
||||
let Some(database) = TestDatabase::create("quota_warmup_claim").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
repository
|
||||
.insert_provider_account(account("acct_warmup", "fixture"))
|
||||
.await
|
||||
.unwrap();
|
||||
let id = ProviderAccountId::new("acct_warmup").unwrap();
|
||||
let current = repository.get_account(&id).await.unwrap().unwrap();
|
||||
let initial = repository
|
||||
.load_quota_warmup_state(¤t)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(initial.version, 0);
|
||||
let claim = || QuotaWarmupStateUpdate {
|
||||
account: ¤t,
|
||||
expected_version: 0,
|
||||
expected_quota_observed_at: None,
|
||||
data: json!({"claimed": true}).as_object().unwrap().clone(),
|
||||
};
|
||||
let (one, two) = tokio::join!(
|
||||
repository.compare_and_swap_quota_warmup_state(claim()),
|
||||
repository.compare_and_swap_quota_warmup_state(claim())
|
||||
);
|
||||
assert_ne!(one.unwrap(), two.unwrap());
|
||||
let reloaded = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let state = reloaded
|
||||
.load_quota_warmup_state(¤t)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(state.version, 1);
|
||||
assert_eq!(state.data["claimed"], true);
|
||||
for mutation in [
|
||||
"update provider_accounts set quota_settings = '{\"codex:604800s\":{\"warmup\":{\"model\":\"test-model\"}}}' where id = 'acct_warmup'",
|
||||
"update provider_accounts set quota_observed_at = now(), updated_at = now(), provider_quota_json = '{}' where id = 'acct_warmup'",
|
||||
"update provider_accounts set enabled = false where id = 'acct_warmup'",
|
||||
"update provider_accounts set credential_state = 'invalid' where id = 'acct_warmup'",
|
||||
"update provider_accounts set credential_revision = credential_revision + 1 where id = 'acct_warmup'",
|
||||
] {
|
||||
sqlx::query(sqlx::AssertSqlSafe(mutation))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!reloaded
|
||||
.compare_and_swap_quota_warmup_state(QuotaWarmupStateUpdate {
|
||||
expected_version: 1,
|
||||
..claim()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
sqlx::query("update provider_accounts set quota_settings = '{}', quota_observed_at = null, provider_quota_json = null, enabled = true, credential_state = $1, credential_revision = $2 where id = 'acct_warmup'")
|
||||
.bind(current.credential_state().as_str()).bind(i64::try_from(current.revision().get()).unwrap())
|
||||
.execute(&database.pool).await.unwrap();
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -355,7 +355,7 @@ async fn disabled_proxy_projects_direct_while_preserving_bindings_and_credential
|
||||
let error = admin
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
account_ids: vec![
|
||||
"acct_enabled_openai".to_owned(),
|
||||
@@ -416,11 +416,11 @@ async fn seed_paused_quota(database: &TestDatabase, id: &str) {
|
||||
"rate_limit": {"primary_window": {"limit_window_seconds": 18000, "used_percent": 99}}
|
||||
});
|
||||
sqlx::query(
|
||||
"update provider_accounts set quota_limits = $2, quota_paused = true,
|
||||
"update provider_accounts set quota_settings = $2, quota_paused = true,
|
||||
provider_quota_json = $3, quota_observed_at = now(), updated_at = now() where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(serde_json::json!({"codex:18000s": 99}))
|
||||
.bind(serde_json::json!({"codex:18000s": {"limitPercent": 99, "warmup": null}}))
|
||||
.bind(window)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
@@ -1219,7 +1219,7 @@ async fn legacy_urls_join_one_catalog_entry_and_invalid_batch_rolls_back() {
|
||||
admin
|
||||
.batch_update_accounts(
|
||||
BatchUpdateAccounts {
|
||||
quota_limits: None,
|
||||
quota_settings: None,
|
||||
auto_use_quota_credits: None,
|
||||
model_access: Default::default(),
|
||||
account_ids: vec!["acct_one".to_owned(), "acct_missing".to_owned()],
|
||||
|
||||
@@ -40,6 +40,7 @@ pub(super) fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettin
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
@@ -113,7 +114,7 @@ async fn privacy_policy_defaults_off_and_round_trips_in_snapshot() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn smart_settings_round_trip_publish_and_restore_database_defaults() {
|
||||
async fn scheduling_settings_round_trip_publish_and_restore_database_defaults() {
|
||||
use gateway_core::account::SmartSchedulingConfig;
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create("smart_config").await else {
|
||||
@@ -126,6 +127,8 @@ async fn smart_settings_round_trip_publish_and_restore_database_defaults() {
|
||||
SmartSchedulingConfig::default()
|
||||
);
|
||||
let mut update = settings_with_margin(3600);
|
||||
let windows = gateway_core::account::ResetWindowWeights::new([0.0, 1.5, 0.0]).unwrap();
|
||||
update.values.quota_reset_window_weights = windows;
|
||||
let expected = serde_json::json!({
|
||||
"loadWeight": 0.0, "quotaWeight": 2.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3,
|
||||
"resetWindowWeights": {"fiveHour":2.0,"weekly":8.0,"monthly":0.0}
|
||||
@@ -137,12 +140,17 @@ async fn smart_settings_round_trip_publish_and_restore_database_defaults() {
|
||||
serde_json::to_value(reloaded.values.smart_scheduling).unwrap(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(reloaded.values.quota_reset_window_weights, windows);
|
||||
let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone())
|
||||
.load_runtime_snapshot()
|
||||
.await
|
||||
.unwrap();
|
||||
let snapshot_values = serde_json::to_value(&snapshot.settings).unwrap();
|
||||
assert_eq!(snapshot_values["smart_scheduling"], expected);
|
||||
assert_eq!(
|
||||
snapshot_values["quota_reset_window_weights"],
|
||||
serde_json::to_value(windows).unwrap()
|
||||
);
|
||||
assert!(snapshot.config_revision > before.config_revision);
|
||||
sqlx::query("update runtime_settings set smart_scheduling_json = default where id = 1")
|
||||
.execute(&database.pool)
|
||||
|
||||
@@ -24,7 +24,7 @@ gateway-core = { path = "../../gateway-core" }
|
||||
gateway-protocol = { path = "../../gateway-protocol" }
|
||||
getrandom = "0.4.3"
|
||||
hex = "0.4.3"
|
||||
lru = "0.18.3"
|
||||
lru = "0.19.0"
|
||||
# TLS 依赖以官方 Core 锁文件为基线,安全补丁经握手特征核验后精确固定
|
||||
# 包含用于约束传递依赖的直接声明
|
||||
# hyper-util 固定到包含 SOCKS IPv6 修复的正式版
|
||||
|
||||
@@ -206,7 +206,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
) -> gateway_admin::model::accounts::ProviderAccountCapabilities {
|
||||
let oauth = authentication_kind == crate::credential::CODEX_AUTHENTICATION_KIND_OAUTH;
|
||||
gateway_admin::model::accounts::ProviderAccountCapabilities {
|
||||
quota_limits_supported: oauth,
|
||||
quota_settings_supported: oauth,
|
||||
quota_credits: oauth,
|
||||
quota: oauth,
|
||||
quota_refresh: oauth,
|
||||
@@ -760,7 +760,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
Ok(project_quota(
|
||||
snapshot,
|
||||
account.quota().is_exhausted(),
|
||||
account.quota_limits(),
|
||||
account.quota_settings(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1244,7 +1244,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
fn project_quota(
|
||||
snapshot: Option<CodexAccountQuotaSnapshot>,
|
||||
limit_reached: bool,
|
||||
limits: &gateway_core::account::AccountQuotaLimits,
|
||||
limits: &gateway_core::account::AccountQuotaSettings,
|
||||
) -> ProviderQuota {
|
||||
let mut quota = snapshot
|
||||
.map(|s| project_quota_snapshot(s, limits))
|
||||
@@ -1255,7 +1255,7 @@ fn project_quota(
|
||||
|
||||
fn project_quota_snapshot(
|
||||
snapshot: CodexAccountQuotaSnapshot,
|
||||
limits: &gateway_core::account::AccountQuotaLimits,
|
||||
limits: &gateway_core::account::AccountQuotaSettings,
|
||||
) -> ProviderQuota {
|
||||
let mut provider_data = Map::new();
|
||||
provider_data.insert(
|
||||
|
||||
@@ -68,7 +68,7 @@ impl OpenAiAdminProvider {
|
||||
Ok(project_quota(
|
||||
snapshot,
|
||||
request.account.quota.is_exhausted(),
|
||||
&request.account.quota_limits,
|
||||
&request.account.quota_settings,
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 逻辑会话账号绑定与线程传输隔离键的单向派生
|
||||
|
||||
use gateway_core::operation::RawJsonPayload;
|
||||
use gateway_core::operation::{ProtocolPayload, RawJsonPayload};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAlias};
|
||||
use serde_json::{Map, Value};
|
||||
@@ -148,6 +148,38 @@ pub(crate) fn derive_codex_session_affinity(
|
||||
.map(|affinity| affinity.with_follow_only(request.client_account_follow_only))
|
||||
}
|
||||
|
||||
/// Messages 的会话只参与账号绑定,不解释为 OpenAI 轮次或原生续接身份
|
||||
pub(crate) fn derive_messages_session_affinity(
|
||||
payload: &ProtocolPayload,
|
||||
headers: &[gateway_core::engine::middleware::MiddlewareHeader],
|
||||
client_api_key_id: &ClientApiKeyId,
|
||||
) -> Option<CodexSessionAffinity> {
|
||||
if payload.protocol() != gateway_protocol::anthropic::ANTHROPIC_PROTOCOL {
|
||||
return None;
|
||||
}
|
||||
let passthrough = crate::transport::request::decode_passthrough_headers(payload.context());
|
||||
let header = headers
|
||||
.iter()
|
||||
.find(|header| {
|
||||
header
|
||||
.name()
|
||||
.eq_ignore_ascii_case("x-claude-code-session-id")
|
||||
})
|
||||
.map(|header| std::str::from_utf8(header.value()).ok())
|
||||
.unwrap_or_else(|| {
|
||||
passthrough
|
||||
.get("x-claude-code-session-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
});
|
||||
let session = gateway_protocol::anthropic::claude_session_id(payload.body(), header)?;
|
||||
session_affinity(
|
||||
"claude-session",
|
||||
&session,
|
||||
Some(&session),
|
||||
client_api_key_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// 仅供选号使用。缓存键与输入正文均不作为会话、轮次或传输身份。
|
||||
pub(crate) fn derive_cache_preference(
|
||||
request: &CodexResponsesRequest,
|
||||
|
||||
@@ -22,7 +22,8 @@ pub(crate) use affinity::{
|
||||
CodexSessionAffinity, derive_cache_preference, derive_codex_cyber_policy_session_key,
|
||||
derive_codex_endpoint_session_affinity, derive_codex_session_affinity,
|
||||
derive_codex_transport_key, derive_endpoint_affinity_with_headers,
|
||||
derive_live_session_affinity, derive_previous_response_id_hash, derive_turn_alias,
|
||||
derive_live_session_affinity, derive_messages_session_affinity,
|
||||
derive_previous_response_id_hash, derive_turn_alias,
|
||||
};
|
||||
pub(crate) use eligibility::CodexSelectionModel;
|
||||
pub(crate) use oauth::oauth_owner_ref;
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//! 额度桶重置后的延迟预热、使用证据复核和持久化领取
|
||||
|
||||
use super::*;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::account::QuotaWarmupStateUpdate;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const RESET_WARMUP_DELAY_SECONDS: i64 = 5 * 60;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct BucketProgress {
|
||||
reset_at: Option<i64>,
|
||||
completed_reset_at: Option<i64>,
|
||||
last_warmup_at: Option<i64>,
|
||||
retry_at: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct WarmupProgress {
|
||||
#[serde(default)]
|
||||
buckets: BTreeMap<String, BucketProgress>,
|
||||
}
|
||||
|
||||
impl WarmupProgress {
|
||||
fn reconcile(&mut self, account: &ProviderAccount, snapshot: &CodexAccountQuotaSnapshot) {
|
||||
for (key, settings) in account.quota_settings().iter() {
|
||||
if settings.warmup.is_none() {
|
||||
continue;
|
||||
}
|
||||
let Some(window) = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.find(|window| window.is_account_wide() && window.key() == key)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let progress = self.buckets.entry(key.to_owned()).or_default();
|
||||
let observed_reset = window.reset_at().map(|at| at.timestamp());
|
||||
if let Some(reset) = progress.reset_at {
|
||||
let last_use = snapshot
|
||||
.last_used_at
|
||||
.get(window.source())
|
||||
.copied()
|
||||
.into_iter()
|
||||
.chain(progress.last_warmup_at)
|
||||
.max();
|
||||
let newly_used = last_use.is_some_and(|used| used >= reset)
|
||||
|| (observed_reset.is_some_and(|next| next > reset)
|
||||
&& DateTime::<Utc>::from(snapshot.observed_at()).timestamp() >= reset
|
||||
&& window.used_percent().is_some_and(|used| used > 0.0));
|
||||
if newly_used {
|
||||
progress.complete(reset);
|
||||
}
|
||||
}
|
||||
// 空闲桶的 resetAt 可能随查询滚动;已登记的本轮边界不能被新查询推迟
|
||||
if progress.reset_at.is_none()
|
||||
&& let Some(reset) = observed_reset
|
||||
&& progress
|
||||
.completed_reset_at
|
||||
.is_none_or(|completed| reset > completed)
|
||||
{
|
||||
progress.reset_at = Some(reset);
|
||||
progress.retry_at = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn due(&self, account: &ProviderAccount, now: DateTime<Utc>) -> Vec<(String, String, i64)> {
|
||||
account
|
||||
.quota_settings()
|
||||
.iter()
|
||||
.filter_map(|(key, settings)| {
|
||||
let warmup = settings.warmup.as_ref()?;
|
||||
let progress = self.buckets.get(key)?;
|
||||
let reset = progress.reset_at?;
|
||||
(now.timestamp() >= reset.saturating_add(RESET_WARMUP_DELAY_SECONDS)
|
||||
&& progress
|
||||
.retry_at
|
||||
.is_none_or(|retry| now.timestamp() >= retry)
|
||||
&& account.model_access().allows(&warmup.model))
|
||||
.then(|| (key.to_owned(), warmup.model.clone(), reset))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketProgress {
|
||||
fn complete(&mut self, reset: i64) {
|
||||
self.completed_reset_at = Some(self.completed_reset_at.map_or(reset, |old| old.max(reset)));
|
||||
self.reset_at = None;
|
||||
self.retry_at = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexCredentialQuotaService {
|
||||
async fn read_warmup_progress(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
) -> Result<Option<(u64, WarmupProgress)>, CodexCredentialQuotaError> {
|
||||
self.store
|
||||
.load_quota_warmup_state(account)
|
||||
.await?
|
||||
.map(|stored| {
|
||||
serde_json::from_value(Value::Object(stored.data))
|
||||
.map(|progress| (stored.version, progress))
|
||||
.map_err(|_| CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn write_warmup_progress(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
version: u64,
|
||||
progress: &WarmupProgress,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
let data = serde_json::to_value(progress)
|
||||
.map_err(|_| CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
Ok(self
|
||||
.store
|
||||
.compare_and_swap_quota_warmup_state(QuotaWarmupStateUpdate {
|
||||
account,
|
||||
expected_version: version,
|
||||
expected_quota_observed_at: Some(snapshot.observed_at()),
|
||||
data: data
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// 与 Cron 共用同一受租约保护的 worker,每次周期独立于全局 Cron 开关
|
||||
pub async fn execute_bucket_warmup_at(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<CodexWarmupSummary, CodexCredentialQuotaError> {
|
||||
let mut summary = CodexWarmupSummary::default();
|
||||
for account in self.repository.list_for_provider().await? {
|
||||
if !warmup::warmup_account_is_available(&account)
|
||||
|| !account.quota_settings().has_warmup()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
match self.warmup_buckets_for_account(&account, now).await {
|
||||
Ok(Some(true)) => summary.warmed_up += 1,
|
||||
Ok(Some(false)) => summary.failed += 1,
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "OpenAI quota bucket warmup check failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
async fn warmup_buckets_for_account(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Option<bool>, CodexCredentialQuotaError> {
|
||||
let Some(snapshot) = self.read_snapshot_for(account).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some((version, mut progress)) = self.read_warmup_progress(account).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let previous = progress.clone();
|
||||
progress.reconcile(account, &snapshot);
|
||||
let due = progress.due(account, now);
|
||||
// 刷新失败或账号仍不可用时最多每五分钟复查,普通轮询不反复访问上游
|
||||
for (key, _, _) in &due {
|
||||
progress
|
||||
.buckets
|
||||
.get_mut(key)
|
||||
.expect("已登记额度桶")
|
||||
.retry_at = Some(now.timestamp() + RESET_WARMUP_DELAY_SECONDS);
|
||||
}
|
||||
if progress != previous
|
||||
&& !self
|
||||
.write_warmup_progress(account, &snapshot, version, &progress)
|
||||
.await?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if due.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.refresh_account(account.id()).await?;
|
||||
let Some(current) = self.store.get_account(account.id()).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !warmup::warmup_account_is_available(¤t)
|
||||
|| current.quota().is_exhausted()
|
||||
|| current.quota_paused()
|
||||
|| !access_token_is_current(¤t, SystemTime::now())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(snapshot) = self.read_snapshot_for(¤t).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some((version, mut progress)) = self.read_warmup_progress(¤t).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
progress.reconcile(¤t, &snapshot);
|
||||
let selected = due.iter().find(|(key, model, reset)| {
|
||||
current
|
||||
.quota_settings()
|
||||
.iter()
|
||||
.any(|(current_key, settings)| {
|
||||
current_key == key
|
||||
&& settings
|
||||
.warmup
|
||||
.as_ref()
|
||||
.is_some_and(|warmup| warmup.model == *model)
|
||||
})
|
||||
&& progress
|
||||
.buckets
|
||||
.get(key)
|
||||
.is_some_and(|bucket| bucket.reset_at == Some(*reset))
|
||||
&& snapshot.windows().iter().any(|window| {
|
||||
window.is_account_wide() && window.key() == key && !window.limit_reached()
|
||||
})
|
||||
&& current.model_access().allows(model)
|
||||
});
|
||||
let Some((_, model, _)) = selected else {
|
||||
self.write_warmup_progress(¤t, &snapshot, version, &progress)
|
||||
.await?;
|
||||
return Ok(None);
|
||||
};
|
||||
let profile = self
|
||||
.profile
|
||||
.resolve_global_request_profile(self.runtime_policy.as_ref())
|
||||
.await
|
||||
.map_err(CodexCredentialQuotaError::Profile)?;
|
||||
let client = CodexBackendClient::new(
|
||||
self.http.clone(),
|
||||
self.base_url.clone(),
|
||||
self.profile.clone(),
|
||||
)
|
||||
.with_request_profile(profile);
|
||||
// 先持久化领取再发请求;超时、取消或重启后不重放结果不明的推理
|
||||
// 通用窗口共享请求用量,同批到期窗口只发一次轻量请求
|
||||
for (key, _, reset) in &due {
|
||||
if let Some(bucket) = progress.buckets.get_mut(key)
|
||||
&& bucket.reset_at == Some(*reset)
|
||||
{
|
||||
bucket.complete(*reset);
|
||||
}
|
||||
}
|
||||
if !self
|
||||
.write_warmup_progress(¤t, &snapshot, version, &progress)
|
||||
.await?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let succeeded = self.warmup_one(¤t, model, &client).await;
|
||||
if succeeded {
|
||||
self.record_scheduled_warmup_use(¤t, Utc::now())
|
||||
.await?;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(Some(succeeded))
|
||||
}
|
||||
|
||||
pub(super) async fn record_scheduled_warmup_use(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
if !account.quota_settings().has_warmup() {
|
||||
return Ok(());
|
||||
}
|
||||
for _ in 0..QUOTA_MERGE_ATTEMPTS {
|
||||
let Some(snapshot) = self.read_snapshot_for(account).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some((version, mut progress)) = self.read_warmup_progress(account).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
progress.reconcile(account, &snapshot);
|
||||
for (key, settings) in account.quota_settings().iter() {
|
||||
if settings.warmup.is_some()
|
||||
&& snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.any(|window| window.is_account_wide() && window.key() == key)
|
||||
{
|
||||
progress
|
||||
.buckets
|
||||
.entry(key.to_owned())
|
||||
.or_default()
|
||||
.last_warmup_at = Some(now.timestamp());
|
||||
}
|
||||
}
|
||||
progress.reconcile(account, &snapshot);
|
||||
if self
|
||||
.write_warmup_progress(account, &snapshot, version, &progress)
|
||||
.await?
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
//! - [`evidence`]:额度接口错误到凭据/额度事实的分类
|
||||
//! - [`recovery`]:各额度窗口独立的恢复基准与访问结论
|
||||
|
||||
mod bucket_warmup;
|
||||
mod document;
|
||||
pub(crate) mod evidence;
|
||||
mod passive;
|
||||
@@ -15,6 +16,8 @@ mod service;
|
||||
pub(crate) mod snapshot;
|
||||
mod warmup;
|
||||
|
||||
const BUCKET_USAGE_FIELD: &str = "_cpr_bucket_last_used_at";
|
||||
|
||||
pub use reset_credits::CodexResetCreditsError;
|
||||
use reset_credits::ResetCreditLocks;
|
||||
use scheduling::CodexQuotaSchedulingProjection;
|
||||
|
||||
@@ -16,11 +16,21 @@ impl CodexCredentialQuotaService {
|
||||
.await
|
||||
}
|
||||
|
||||
/// 把一次推理响应中采集的结构化限流观察合并后单次落库
|
||||
/// 把成功推理响应中采集的结构化限流观察合并后单次落库
|
||||
pub async fn synchronize_passive_rate_limits(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
rate_limits: &[ParsedRateLimits],
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
self.synchronize_passive_rate_limits_with_usage(account, rate_limits, true)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn synchronize_passive_rate_limits_with_usage(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
rate_limits: &[ParsedRateLimits],
|
||||
request_used_quota: bool,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
if account.authentication_kind() != crate::credential::CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
|| rate_limits.is_empty()
|
||||
@@ -61,7 +71,7 @@ impl CodexCredentialQuotaService {
|
||||
if merge_base.observed_at.is_some_and(|at| at >= observed_at) {
|
||||
return Ok(false);
|
||||
}
|
||||
let previous = if account.quota_limits().is_empty() {
|
||||
let previous = if account.quota_settings().is_empty() {
|
||||
None
|
||||
} else {
|
||||
existing.as_ref().and_then(quota_snapshot_from_observation)
|
||||
@@ -75,14 +85,18 @@ impl CodexCredentialQuotaService {
|
||||
return Ok(false);
|
||||
}
|
||||
let prior = existing_state.unwrap_or_else(|| account.quota());
|
||||
let (mut quota, updated_default) = merge_passive_quota(existing, rate_limits);
|
||||
let (mut quota, updated_default) = merge_passive_quota(
|
||||
existing,
|
||||
rate_limits,
|
||||
request_used_quota.then_some(observed_at),
|
||||
);
|
||||
let mut snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota.clone()),
|
||||
)?;
|
||||
snapshot.retain_limit_evidence(account.quota_limits(), previous.as_ref(), &mut quota);
|
||||
snapshot.retain_limit_evidence(account.quota_settings(), previous.as_ref(), &mut quota);
|
||||
let credit_access = prior.evidence() == Some(QuotaEvidence::CreditsAvailable);
|
||||
let fresh_credits = rate_limits.iter().any(|limits| limits.credits.is_some());
|
||||
// 具名桶和元数据不改变套餐结论;点数不能再用时,仍须撤销原先仅凭点数的放行
|
||||
@@ -114,7 +128,7 @@ impl CodexCredentialQuotaService {
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
quota_pause: Some(
|
||||
snapshot.pause_decision(account.quota_limits(), Some(merge_base)),
|
||||
snapshot.pause_decision(account.quota_settings(), Some(merge_base)),
|
||||
),
|
||||
expected_connection: None,
|
||||
plan_type,
|
||||
@@ -140,6 +154,7 @@ impl CodexCredentialQuotaService {
|
||||
fn merge_passive_quota(
|
||||
mut quota: Map<String, Value>,
|
||||
observations: &[ParsedRateLimits],
|
||||
used_at: Option<SystemTime>,
|
||||
) -> (Map<String, Value>, bool) {
|
||||
let mut updated_default = false;
|
||||
let mut snapshots = RateLimitSnapshotsByLimitId::take_from_document(&mut quota);
|
||||
@@ -174,6 +189,19 @@ fn merge_passive_quota(
|
||||
})
|
||||
.or_else(|| resolved_limit_ids.get(DEFAULT_CODEX_LIMIT_ID).cloned());
|
||||
if let Some(active_limit) = active_limit {
|
||||
if let Some(used_at) = used_at {
|
||||
let usage = quota
|
||||
.entry(super::BUCKET_USAGE_FIELD)
|
||||
.or_insert_with(|| serde_json::json!({}));
|
||||
if let Some(usage) = usage.as_object_mut() {
|
||||
usage.insert(
|
||||
active_limit.clone(),
|
||||
serde_json::json!(
|
||||
chrono::DateTime::<chrono::Utc>::from(used_at).timestamp()
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
quota.insert("active_limit".to_owned(), Value::String(active_limit));
|
||||
}
|
||||
merge_passive_metadata(&mut quota, rate_limits);
|
||||
|
||||
@@ -352,7 +352,7 @@ fn quota_refresh_candidate(
|
||||
.filter(|window| {
|
||||
window.is_account_wide()
|
||||
&& account
|
||||
.quota_limits()
|
||||
.quota_settings()
|
||||
.reached(window.key(), window.used_percent())
|
||||
})
|
||||
.filter_map(CodexQuotaWindow::reset_at)
|
||||
|
||||
@@ -471,6 +471,7 @@ impl CodexCredentialQuotaService {
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
object.remove(RECOVERY_FIELD);
|
||||
object.remove(BUCKET_USAGE_FIELD);
|
||||
object.remove(LIMIT_EVIDENCE_FIELD);
|
||||
let mut snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
@@ -478,16 +479,21 @@ impl CodexCredentialQuotaService {
|
||||
observed_at,
|
||||
&Value::Object(object.clone()),
|
||||
)?;
|
||||
let (merge_base, previous) =
|
||||
if account.quota().is_exhausted() || !account.quota_limits().is_empty() {
|
||||
let (base, snapshot) = self.read_snapshot_with_base(account).await?;
|
||||
if base.observed_at.is_some_and(|at| at >= observed_at) {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
(Some(base), snapshot)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
// 请求使用证据与上游比例分开保留,0% 取整不能让主动刷新抹掉已发生的请求
|
||||
let (base, previous) = self.read_snapshot_with_base(account).await?;
|
||||
if base.observed_at.is_some_and(|at| at >= observed_at) {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
let merge_base = Some(base);
|
||||
if let Some(previous) = &previous {
|
||||
snapshot.last_used_at.clone_from(&previous.last_used_at);
|
||||
if !snapshot.last_used_at.is_empty() {
|
||||
object.insert(
|
||||
BUCKET_USAGE_FIELD.into(),
|
||||
serde_json::json!(snapshot.last_used_at),
|
||||
);
|
||||
}
|
||||
}
|
||||
// 失败后回读只允许明确可用的点数恢复访问,套餐 allowed 仍可能滞后
|
||||
let observe_access = matches!(authority, QuotaRefreshAuthority::ObserveAccess)
|
||||
|| (account.auto_use_quota_credits() && snapshot.credits_usable());
|
||||
@@ -515,7 +521,11 @@ impl CodexCredentialQuotaService {
|
||||
.with_observed_at(observed_at),
|
||||
});
|
||||
}
|
||||
snapshot.retain_limit_evidence(account.quota_limits(), previous.as_ref(), &mut object);
|
||||
snapshot.retain_limit_evidence(
|
||||
account.quota_settings(),
|
||||
previous.as_ref(),
|
||||
&mut object,
|
||||
);
|
||||
let state = if matches!(authority, QuotaRefreshAuthority::PreserveAccess) {
|
||||
snapshot.access_with_credits(account.quota(), account.auto_use_quota_credits())
|
||||
} else {
|
||||
@@ -531,7 +541,9 @@ impl CodexCredentialQuotaService {
|
||||
if self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
quota_pause: Some(snapshot.pause_decision(account.quota_limits(), merge_base)),
|
||||
quota_pause: Some(
|
||||
snapshot.pause_decision(account.quota_settings(), merge_base),
|
||||
),
|
||||
expected_connection: None,
|
||||
plan_type: observed_account_plan(account.plan_type(), snapshot.plan_type()),
|
||||
account_id: account.id().clone(),
|
||||
|
||||
@@ -122,13 +122,14 @@ pub struct CodexAccountQuotaSnapshot {
|
||||
quota: QuotaState,
|
||||
windows: Vec<CodexQuotaWindow>,
|
||||
unconfirmed_limits: BTreeMap<String, f64>,
|
||||
pub(super) last_used_at: BTreeMap<String, i64>,
|
||||
pub(super) recovery: Option<QuotaRecovery>,
|
||||
}
|
||||
|
||||
impl CodexAccountQuotaSnapshot {
|
||||
pub(super) fn pause_decision(
|
||||
&self,
|
||||
limits: &gateway_core::account::AccountQuotaLimits,
|
||||
limits: &gateway_core::account::AccountQuotaSettings,
|
||||
merge_base: Option<super::QuotaMergeBase>,
|
||||
) -> gateway_core::account::QuotaPauseDecision {
|
||||
gateway_core::account::QuotaPauseDecision {
|
||||
@@ -155,7 +156,7 @@ impl CodexAccountQuotaSnapshot {
|
||||
/// 迟到的在途响应或不完整的刷新不能清除已确认的上限,只有对应窗口的新比例才能覆盖它
|
||||
pub(super) fn retain_limit_evidence(
|
||||
&mut self,
|
||||
limits: &gateway_core::account::AccountQuotaLimits,
|
||||
limits: &gateway_core::account::AccountQuotaSettings,
|
||||
previous: Option<&Self>,
|
||||
document: &mut Map<String, Value>,
|
||||
) {
|
||||
@@ -441,6 +442,16 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
account_id,
|
||||
credential_revision,
|
||||
observed_at,
|
||||
last_used_at: object
|
||||
.get(super::BUCKET_USAGE_FIELD)
|
||||
.and_then(Value::as_object)
|
||||
.map(|usage| {
|
||||
usage
|
||||
.iter()
|
||||
.filter_map(|(source, at)| Some((source.clone(), at.as_i64()?)))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
plan_type: object
|
||||
.get("plan_type")
|
||||
.and_then(Value::as_str)
|
||||
|
||||
@@ -8,7 +8,7 @@ enum WarmupDecision<'a> {
|
||||
Exhausted(Option<&'a CodexQuotaWindow>),
|
||||
}
|
||||
|
||||
fn warmup_account_is_available(account: &ProviderAccount) -> bool {
|
||||
pub(super) fn warmup_account_is_available(account: &ProviderAccount) -> bool {
|
||||
account.authentication_kind() == crate::credential::CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
&& account.enabled()
|
||||
&& matches!(
|
||||
@@ -194,126 +194,140 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
}
|
||||
|
||||
let credential = match self.repository.load_runtime_credential(&account).await {
|
||||
Ok(cred) => cred,
|
||||
Err(_) => {
|
||||
summary.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let authorization = match credential.authentication.authorization_header() {
|
||||
Ok(auth) => auth,
|
||||
Err(_) => {
|
||||
summary.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let mut body = Map::new();
|
||||
body.insert("model".to_owned(), Value::String(model.to_owned()));
|
||||
body.insert(
|
||||
"input".to_owned(),
|
||||
serde_json::json!([{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "hello"}]
|
||||
}]),
|
||||
);
|
||||
body.insert("stream".to_owned(), Value::Bool(true));
|
||||
body.insert("store".to_owned(), Value::Bool(false));
|
||||
body.insert(
|
||||
"service_tier".to_owned(),
|
||||
Value::String("default".to_owned()),
|
||||
);
|
||||
// 目录里各模型的最低档都是 low,none 会收到上游 error 终态并让整次预热记为失败
|
||||
body.insert("reasoning".to_owned(), serde_json::json!({"effort": "low"}));
|
||||
body.insert("text".to_owned(), serde_json::json!({"verbosity": "low"}));
|
||||
|
||||
let upstream_request = CodexResponsesRequest::from_body(body);
|
||||
let request_id = format!("warmup_{}", Uuid::now_v7().simple());
|
||||
let client_for_account = match client.for_account(&account) {
|
||||
Ok(c) => c,
|
||||
Err(error) => {
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "warmup client for account failed");
|
||||
summary.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let context = crate::transport::CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
account.upstream_account_id(),
|
||||
&request_id,
|
||||
None,
|
||||
);
|
||||
|
||||
// HTTP 200 只确认响应头;必须消费 SSE 直到终态才能确认预热结果和额度事件
|
||||
let attempt = tokio::time::timeout(WARMUP_REQUEST_TIMEOUT, async {
|
||||
let mut response = client_for_account
|
||||
.create_response_stream_http_sse(&upstream_request, context)
|
||||
.await
|
||||
.map_err(|_| "request_rejected")?;
|
||||
let terminal = consume_warmup_sse(&mut response).await;
|
||||
let rate_limit_updates = match response.rate_limit_updates.as_ref() {
|
||||
Some(updates) => std::mem::take(&mut *updates.lock().await),
|
||||
None => Vec::new(),
|
||||
};
|
||||
Ok::<_, &'static str>((terminal, response.rate_limit_headers, rate_limit_updates))
|
||||
})
|
||||
.await;
|
||||
match attempt {
|
||||
Ok(Ok((terminal, headers, rate_limit_updates))) => {
|
||||
match terminal {
|
||||
Ok(()) => {
|
||||
// 被动额度同步会把成功请求的窗口事实标为可用,失败流不能复用该结论
|
||||
if !headers.is_empty()
|
||||
&& let Err(error) =
|
||||
self.synchronize_passive_headers(&account, &headers).await
|
||||
{
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "OpenAI warmup rate-limit header sync failed");
|
||||
}
|
||||
if !rate_limit_updates.is_empty()
|
||||
&& let Err(error) = self
|
||||
.synchronize_passive_rate_limits(&account, &rate_limit_updates)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "OpenAI warmup rate-limit event sync failed");
|
||||
}
|
||||
summary.warmed_up += 1;
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
model,
|
||||
"OpenAI account warmed up successfully"
|
||||
);
|
||||
}
|
||||
Err(reason) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason,
|
||||
"OpenAI account warmup stream failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(reason)) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason,
|
||||
"OpenAI account warmup request rejected by upstream"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
"OpenAI account warmup request timed out"
|
||||
);
|
||||
}
|
||||
if self.warmup_one(&account, model, &client).await {
|
||||
summary.warmed_up += 1;
|
||||
self.record_scheduled_warmup_use(&account, chrono::Utc::now())
|
||||
.await?;
|
||||
} else {
|
||||
summary.failed += 1;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
pub(super) async fn warmup_one(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
model: &str,
|
||||
client: &CodexBackendClient,
|
||||
) -> bool {
|
||||
let mut summary = CodexWarmupSummary::default();
|
||||
let credential = match self.repository.load_runtime_credential(account).await {
|
||||
Ok(cred) => cred,
|
||||
Err(_) => {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let authorization = match credential.authentication.authorization_header() {
|
||||
Ok(auth) => auth,
|
||||
Err(_) => {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
let mut body = Map::new();
|
||||
body.insert("model".to_owned(), Value::String(model.to_owned()));
|
||||
body.insert(
|
||||
"input".to_owned(),
|
||||
serde_json::json!([{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "hello"}]
|
||||
}]),
|
||||
);
|
||||
body.insert("stream".to_owned(), Value::Bool(true));
|
||||
body.insert("store".to_owned(), Value::Bool(false));
|
||||
body.insert(
|
||||
"service_tier".to_owned(),
|
||||
Value::String("default".to_owned()),
|
||||
);
|
||||
// 目录里各模型的最低档都是 low,none 会收到上游 error 终态并让整次预热记为失败
|
||||
body.insert("reasoning".to_owned(), serde_json::json!({"effort": "low"}));
|
||||
body.insert("text".to_owned(), serde_json::json!({"verbosity": "low"}));
|
||||
|
||||
let upstream_request = CodexResponsesRequest::from_body(body);
|
||||
let request_id = format!("warmup_{}", Uuid::now_v7().simple());
|
||||
let client_for_account = match client.for_account(account) {
|
||||
Ok(c) => c,
|
||||
Err(error) => {
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "warmup client for account failed");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let context = crate::transport::CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
account.upstream_account_id(),
|
||||
&request_id,
|
||||
None,
|
||||
);
|
||||
|
||||
// HTTP 200 只确认响应头;必须消费 SSE 直到终态才能确认预热结果和额度事件
|
||||
let attempt = tokio::time::timeout(WARMUP_REQUEST_TIMEOUT, async {
|
||||
let mut response = client_for_account
|
||||
.create_response_stream_http_sse(&upstream_request, context)
|
||||
.await
|
||||
.map_err(|_| "request_rejected")?;
|
||||
let terminal = consume_warmup_sse(&mut response).await;
|
||||
let rate_limit_updates = match response.rate_limit_updates.as_ref() {
|
||||
Some(updates) => std::mem::take(&mut *updates.lock().await),
|
||||
None => Vec::new(),
|
||||
};
|
||||
Ok::<_, &'static str>((terminal, response.rate_limit_headers, rate_limit_updates))
|
||||
})
|
||||
.await;
|
||||
match attempt {
|
||||
Ok(Ok((terminal, headers, rate_limit_updates))) => {
|
||||
match terminal {
|
||||
Ok(()) => {
|
||||
// 被动额度同步会把成功请求的窗口事实标为可用,失败流不能复用该结论
|
||||
if !headers.is_empty()
|
||||
&& let Err(error) =
|
||||
self.synchronize_passive_headers(account, &headers).await
|
||||
{
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "OpenAI warmup rate-limit header sync failed");
|
||||
}
|
||||
if !rate_limit_updates.is_empty()
|
||||
&& let Err(error) = self
|
||||
.synchronize_passive_rate_limits(account, &rate_limit_updates)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(account_id = %account.id(), error = %error, "OpenAI warmup rate-limit event sync failed");
|
||||
}
|
||||
summary.warmed_up += 1;
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
model,
|
||||
"OpenAI account warmed up successfully"
|
||||
);
|
||||
}
|
||||
Err(reason) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason,
|
||||
"OpenAI account warmup stream failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(reason)) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason,
|
||||
"OpenAI account warmup request rejected by upstream"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
"OpenAI account warmup request timed out"
|
||||
);
|
||||
}
|
||||
}
|
||||
summary.warmed_up == 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -499,7 +499,9 @@ impl CodexCredentialSelector {
|
||||
&& request.attempt.required_account().is_none()
|
||||
&& matches!(
|
||||
request.attempt.continuation_attempt(),
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny
|
||||
ContinuationAttempt::None
|
||||
| ContinuationAttempt::ReplayAny
|
||||
| ContinuationAttempt::RotateForClientReplay
|
||||
) =>
|
||||
{
|
||||
reason
|
||||
@@ -611,7 +613,9 @@ impl CodexCredentialSelector {
|
||||
.account_state_owner()
|
||||
.filter(|owner| owner.provider() == &self.provider_kind)
|
||||
.map(|owner| owner.account().clone()),
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny => None,
|
||||
ContinuationAttempt::None
|
||||
| ContinuationAttempt::ReplayAny
|
||||
| ContinuationAttempt::RotateForClientReplay => None,
|
||||
};
|
||||
let required_account = request.attempt.required_account().cloned();
|
||||
if required_account
|
||||
@@ -1112,6 +1116,12 @@ impl CodexCredentialSelector {
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if !diagnostic {
|
||||
for candidate in &candidates {
|
||||
if request
|
||||
.attempt
|
||||
.excludes_proxy(candidate.account.outbound_proxy())
|
||||
{
|
||||
excluded.insert(candidate.account.id().clone());
|
||||
}
|
||||
if candidate.account.quota_paused() {
|
||||
if follow_only
|
||||
&& binding.as_ref().is_some_and(|binding| {
|
||||
@@ -1280,6 +1290,16 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
let Some(selection) = selection else {
|
||||
if follow_only && let Some(binding) = binding.as_ref() {
|
||||
// 如果整个候选池共用本请求已排除的入口,等待根线程迁移也没有可用出口
|
||||
if !candidates.is_empty()
|
||||
&& candidates.iter().all(|candidate| {
|
||||
request
|
||||
.attempt
|
||||
.excludes_proxy(candidate.account.outbound_proxy())
|
||||
})
|
||||
{
|
||||
return Err(CredentialSelectionError::NoEligibleCredential);
|
||||
}
|
||||
// 冷却、排除、停用或当前账号不在本次范围时也只等待根线程恢复或迁移
|
||||
waiting
|
||||
.wait(std::slice::from_ref(binding.account_id()))
|
||||
|
||||
@@ -721,6 +721,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
&active_account,
|
||||
&response.rate_limit_headers,
|
||||
image_endpoint,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
if !response.set_cookie_headers.is_empty()
|
||||
@@ -906,7 +907,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
apply_failure(&failure_context, &active_account, &failure)
|
||||
.await;
|
||||
Err(quota_continuation_replay_error(
|
||||
Err(continuation_recovery_error(
|
||||
failure.error,
|
||||
&request,
|
||||
ReplayBoundary::BeforeSemanticOutput,
|
||||
@@ -1023,6 +1024,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
"a,
|
||||
&active_account,
|
||||
passive_quota_observation.rate_limits(),
|
||||
decoder.semantic_output_seen(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1107,12 +1109,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
"a,
|
||||
&active_account,
|
||||
passive_quota_observation.rate_limits(),
|
||||
decoder.semantic_output_seen(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
apply_failure(&failure_context, &active_account, &failure)
|
||||
.await;
|
||||
Err(quota_continuation_replay_error(
|
||||
Err(continuation_recovery_error(
|
||||
failure.error,
|
||||
&request,
|
||||
ReplayBoundary::from_semantic_output(client_events.replay_closed()),
|
||||
@@ -1211,6 +1214,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
"a,
|
||||
&active_account,
|
||||
passive_quota_observation.rate_limits(),
|
||||
completed && terminal_failure.is_none() || decoder.semantic_output_seen(),
|
||||
)
|
||||
.await;
|
||||
trace.record("provider.quota_sync", json!({
|
||||
@@ -1267,7 +1271,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.error
|
||||
.with_atomic_client_events(events);
|
||||
}
|
||||
Err(quota_continuation_replay_error(
|
||||
Err(continuation_recovery_error(
|
||||
failure.error,
|
||||
&request,
|
||||
ReplayBoundary::from_semantic_output(failure_after_commit),
|
||||
@@ -1331,11 +1335,16 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
passive_quota_observation.observe(&updates);
|
||||
observation_state.merge_rate_limit_headers(&rate_limit_update_headers(&updates))
|
||||
};
|
||||
let completed = events
|
||||
.iter()
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
if allows_account_state_mutation {
|
||||
synchronize_passive_quota(
|
||||
"a,
|
||||
&active_account,
|
||||
passive_quota_observation.rate_limits(),
|
||||
completed && terminal_failure.is_none() || decoder.semantic_output_seen(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1351,10 +1360,6 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
attach_openai_session_update(&mut events, &mut session_capture);
|
||||
let completed = events
|
||||
.iter()
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
let terminal_changed = completed
|
||||
&& observation_state.mark_completed(terminal_response_is_incomplete(&events));
|
||||
if response_transport == CodexBackendTransport::WebSocket
|
||||
@@ -1396,7 +1401,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.error
|
||||
.with_atomic_client_events(events);
|
||||
}
|
||||
Err(quota_continuation_replay_error(
|
||||
Err(continuation_recovery_error(
|
||||
failure.error,
|
||||
&request,
|
||||
ReplayBoundary::from_semantic_output(failure_after_commit),
|
||||
|
||||
@@ -317,6 +317,7 @@ pub(super) async fn apply_failure(
|
||||
account,
|
||||
&failure.rate_limit_headers,
|
||||
context.image_endpoint,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
if failure.model_unsupported
|
||||
@@ -441,6 +442,7 @@ pub(super) fn schedule_authoritative_quota_refresh_after_failure(
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum WebSocketFailurePolicy {
|
||||
Budgeted,
|
||||
TransientConnect,
|
||||
ImmediateFallback,
|
||||
}
|
||||
|
||||
@@ -469,6 +471,9 @@ pub(super) fn websocket_client_failure_policy(
|
||||
} if *status == reqwest::StatusCode::UPGRADE_REQUIRED => {
|
||||
Some(WebSocketFailurePolicy::ImmediateFallback)
|
||||
}
|
||||
CodexClientError::WebSocket(error) if transient_websocket_connect(error) => {
|
||||
Some(WebSocketFailurePolicy::TransientConnect)
|
||||
}
|
||||
CodexClientError::WebSocket(error)
|
||||
if !matches!(
|
||||
error.classified(),
|
||||
@@ -484,6 +489,28 @@ pub(super) fn websocket_client_failure_policy(
|
||||
}
|
||||
}
|
||||
|
||||
fn transient_websocket_connect(error: &CodexWebSocketExchangeError) -> bool {
|
||||
use std::io::ErrorKind;
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. } => true,
|
||||
CodexWebSocketExchangeError::Connect {
|
||||
source: tungstenite::Error::Io(io),
|
||||
..
|
||||
} => matches!(
|
||||
io.kind(),
|
||||
ErrorKind::ConnectionRefused
|
||||
| ErrorKind::NetworkUnreachable
|
||||
| ErrorKind::HostUnreachable
|
||||
| ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::BrokenPipe
|
||||
| ErrorKind::UnexpectedEof
|
||||
| ErrorKind::TimedOut
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn apply_websocket_recovery_policy(
|
||||
failure: &mut MappedProviderFailure,
|
||||
context: WebSocketRecoveryContext<'_>,
|
||||
@@ -544,6 +571,15 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|
||||
|| context.retry_count >= context.max_retries
|
||||
|| session_budget_exhausted;
|
||||
if fallback_now && !context.requirement.allows_pre_send_http_fallback() {
|
||||
return;
|
||||
}
|
||||
// 只为无续接依赖的新请求保留重选机会;是否重试 WS 或退回 HTTP 仍由协议预算决定
|
||||
if context.policy == WebSocketFailurePolicy::TransientConnect
|
||||
&& context.requirement.allows_connection_restart()
|
||||
{
|
||||
failure.error.allow_connection_rotation();
|
||||
}
|
||||
if !fallback_now {
|
||||
let Some(retry_index) = context.retry_count.checked_add(1).and_then(NonZeroU32::new) else {
|
||||
return;
|
||||
@@ -573,14 +609,11 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
upstream_send_state = ?failure.error.send_state(),
|
||||
replay_proved_safe = failure.error.replay_is_safe(),
|
||||
transport_requirement = context.requirement.as_str(),
|
||||
"OpenAI upstream WebSocket failed; retrying the same account"
|
||||
"OpenAI upstream WebSocket failed; requesting transport retry"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if !context.requirement.allows_pre_send_http_fallback() {
|
||||
return;
|
||||
}
|
||||
failure.error.set_pre_delivery_transport_fallback();
|
||||
if let Some(key) = context.session_transport_key {
|
||||
context.session_transport_recovery.disable_websocket(key);
|
||||
@@ -600,7 +633,8 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
websocket_retry_count = context.retry_count,
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_fallback_reason = match context.policy {
|
||||
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
|
||||
WebSocketFailurePolicy::Budgeted | WebSocketFailurePolicy::TransientConnect =>
|
||||
"retry_budget_exhausted",
|
||||
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
|
||||
},
|
||||
session_affinity_present = context.session_transport_key.is_some(),
|
||||
@@ -637,17 +671,17 @@ pub(super) fn continuation_replay_required_error(reason: &'static str) -> Provid
|
||||
"previous response is unavailable in the selected upstream scope; client replay is required",
|
||||
))
|
||||
.with_client_visible_upstream_error(continuation_replay_error_detail())
|
||||
.with_client_visible_upstream_response(continuation_replay_response(Vec::new()))
|
||||
}
|
||||
|
||||
/// 额度拒绝使当前账号无法继续原生历史,由客户端携带完整输入创建新链
|
||||
/// 原生历史恢复由客户端携带完整输入创建新链;容量失败先交给 Core 决定是否迁移账号
|
||||
/// 只投影客户端响应;真实上游分类、发送状态和错误正文继续用于隔离与记账
|
||||
pub(super) fn quota_continuation_replay_error(
|
||||
pub(super) fn continuation_recovery_error(
|
||||
mut error: ProviderError,
|
||||
request: &CodexResponsesRequest,
|
||||
replay_boundary: ReplayBoundary,
|
||||
) -> ProviderError {
|
||||
if request.previous_response_id().is_none()
|
||||
|| error.kind() != ProviderErrorKind::QuotaExhausted
|
||||
|| !error.replay_is_safe()
|
||||
|| error.send_state() == UpstreamSendState::Ambiguous
|
||||
|| !replay_boundary.permits_provider_proof()
|
||||
@@ -655,6 +689,15 @@ pub(super) fn quota_continuation_replay_error(
|
||||
return error;
|
||||
}
|
||||
|
||||
if error.kind() == ProviderErrorKind::UpstreamCapacityUnavailable {
|
||||
return error.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
);
|
||||
}
|
||||
if error.kind() != ProviderErrorKind::QuotaExhausted {
|
||||
return error;
|
||||
}
|
||||
|
||||
let mut headers: Vec<ProviderResponseHeader> = error
|
||||
.client_visible_upstream_response()
|
||||
.map(|response| {
|
||||
@@ -662,7 +705,10 @@ pub(super) fn quota_continuation_replay_error(
|
||||
.headers()
|
||||
.iter()
|
||||
// 额度窗口的等待时间不适用于客户端重建历史
|
||||
.filter(|header| !header.name().eq_ignore_ascii_case("retry-after"))
|
||||
.filter(|header| {
|
||||
!header.name().eq_ignore_ascii_case("retry-after")
|
||||
&& !header.name().eq_ignore_ascii_case("retry-after-ms")
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
})
|
||||
@@ -678,12 +724,6 @@ pub(super) fn quota_continuation_replay_error(
|
||||
Bytes::copy_from_slice(request_id.as_str().as_bytes()),
|
||||
));
|
||||
}
|
||||
let detail = continuation_replay_error_detail();
|
||||
let body = json!({"error": {
|
||||
"message": detail.message(),
|
||||
"code": detail.code(),
|
||||
"type": detail.error_type(),
|
||||
}});
|
||||
// 丢弃仍未交付的前导与额度错误帧,避免原错误覆盖客户端恢复投影
|
||||
// 已交付的 Lite 前导不在此批次内,语义输出后的失败不进入此分支
|
||||
drop(error.take_atomic_client_events());
|
||||
@@ -693,15 +733,25 @@ pub(super) fn quota_continuation_replay_error(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
)
|
||||
.with_continuation_unavailable_reason("quota_exhausted")
|
||||
.with_client_visible_upstream_error(detail)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
reqwest::StatusCode::BAD_REQUEST.as_u16(),
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from(body.to_string()),
|
||||
)
|
||||
.with_headers(headers),
|
||||
)
|
||||
.with_client_visible_upstream_error(continuation_replay_error_detail())
|
||||
.with_client_visible_upstream_response(continuation_replay_response(headers))
|
||||
}
|
||||
|
||||
fn continuation_replay_response(
|
||||
headers: Vec<ProviderResponseHeader>,
|
||||
) -> ClientVisibleUpstreamResponse {
|
||||
let detail = continuation_replay_error_detail();
|
||||
let body = json!({"error": {
|
||||
"message": detail.message(),
|
||||
"code": detail.code(),
|
||||
"type": detail.error_type(),
|
||||
}});
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
reqwest::StatusCode::BAD_REQUEST.as_u16(),
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from(body.to_string()),
|
||||
)
|
||||
.with_headers(headers)
|
||||
}
|
||||
|
||||
fn continuation_replay_error_detail() -> ClientVisibleUpstreamError {
|
||||
@@ -971,6 +1021,11 @@ pub(super) fn map_client_error(
|
||||
);
|
||||
}
|
||||
if let Some(diagnostic) = diagnostic {
|
||||
if diagnostic.proxy_hint() == Some(ProxyConnectionHint::Unreachable)
|
||||
&& let Some(proxy) = handshake_account.and_then(ProviderAccount::outbound_proxy)
|
||||
{
|
||||
failure.error = failure.error.with_unreachable_proxy(proxy.dial_endpoint());
|
||||
}
|
||||
failure.error = failure.error.with_diagnostic(diagnostic);
|
||||
}
|
||||
if let Some(raw) = raw_upstream_error {
|
||||
@@ -1061,7 +1116,11 @@ pub(super) fn client_diagnostic(
|
||||
.with_classification(stage, code)
|
||||
.with_io_cause(error);
|
||||
return Some(if uses_account_proxy && error.is_connect() {
|
||||
diagnostic.with_proxy_hint(ProxyConnectionHint::CheckProxy)
|
||||
diagnostic.with_proxy_hint(if http_proxy_dial_failed(error) {
|
||||
ProxyConnectionHint::Unreachable
|
||||
} else {
|
||||
ProxyConnectionHint::CheckProxy
|
||||
})
|
||||
} else {
|
||||
diagnostic
|
||||
});
|
||||
@@ -1740,6 +1799,43 @@ pub(super) fn provider_error(
|
||||
ProviderError::new(kind, send_state)
|
||||
}
|
||||
|
||||
/// 账号代理覆盖所有 HTTP 目的地。只有到代理的 TCP 拨号失败可排除整个入口;
|
||||
/// SOCKS 握手和 CONNECT 隧道返回的错误均不能证明入口不可达
|
||||
fn http_proxy_dial_failed(error: &reqwest::Error) -> bool {
|
||||
use hyper_util::client::legacy::connect::{HttpConnector, proxy::SocksV5};
|
||||
use std::error::Error;
|
||||
use tower_service::Service;
|
||||
|
||||
type TcpError = <HttpConnector as Service<hyper::Uri>>::Error;
|
||||
type SocksError = <SocksV5<HttpConnector> as Service<hyper::Uri>>::Error;
|
||||
let mut source = error.source();
|
||||
while let Some(cause) = source {
|
||||
let dial =
|
||||
cause
|
||||
.downcast_ref::<TcpError>()
|
||||
.or_else(|| match cause.downcast_ref::<SocksError>() {
|
||||
Some(SocksError::Inner(dial)) => Some(dial),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(dial) = dial {
|
||||
return dial
|
||||
.source()
|
||||
.and_then(|inner| inner.downcast_ref::<std::io::Error>())
|
||||
.is_some_and(|io| {
|
||||
matches!(
|
||||
io.kind(),
|
||||
std::io::ErrorKind::ConnectionRefused
|
||||
| std::io::ErrorKind::TimedOut
|
||||
| std::io::ErrorKind::NetworkUnreachable
|
||||
| std::io::ErrorKind::HostUnreachable
|
||||
)
|
||||
});
|
||||
}
|
||||
source = cause.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// 只恢复明确未发送的 HTTP 建连失败;TLS 协议、证书及配置错误不反复尝试同一路径
|
||||
fn transient_http_connect(error: &reqwest::Error) -> bool {
|
||||
if !error.is_connect() || error.is_builder() {
|
||||
|
||||
@@ -30,6 +30,26 @@ impl CodexProvider {
|
||||
if !matches!(source, OPENAI_PROTOCOL | ANTHROPIC_PROTOCOL) {
|
||||
return Err(invalid_request());
|
||||
}
|
||||
if !context.is_diagnostic_required_account()
|
||||
&& (source == ANTHROPIC_PROTOCOL
|
||||
|| selected.lease.authentication().api_protocol() == ApiKeyProtocol::Messages)
|
||||
{
|
||||
// attempt 改写后的身份必须在跨协议转换前复验,避免绕过已有账号绑定
|
||||
let affinity = crate::credential::derive_messages_session_affinity(
|
||||
generate.protocol_payload(),
|
||||
&headers,
|
||||
context.client_api_key_ref(),
|
||||
);
|
||||
self.selector
|
||||
.validate_translated_selection(
|
||||
&mut selected.lease,
|
||||
affinity.as_ref(),
|
||||
None,
|
||||
context.account_selection_policy(),
|
||||
)
|
||||
.await
|
||||
.map_err(map_selection_error)?;
|
||||
}
|
||||
if selected.lease.authentication().api_protocol() == ApiKeyProtocol::Messages {
|
||||
if context.continuation_attempt() == ContinuationAttempt::Native
|
||||
|| generate.provider_session_state(PROVIDER_NAME).is_some()
|
||||
@@ -124,21 +144,10 @@ impl CodexProvider {
|
||||
selected: SelectedGenerate,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let SelectedGenerate {
|
||||
mut lease,
|
||||
lease,
|
||||
account_selection_wait_ms,
|
||||
..
|
||||
} = selected;
|
||||
if !context.is_diagnostic_required_account() {
|
||||
self.selector
|
||||
.validate_translated_selection(
|
||||
&mut lease,
|
||||
None,
|
||||
None,
|
||||
context.account_selection_policy(),
|
||||
)
|
||||
.await
|
||||
.map_err(map_selection_error)?;
|
||||
}
|
||||
self.messages_stream(
|
||||
body,
|
||||
source_context,
|
||||
|
||||
@@ -673,8 +673,7 @@ impl Provider for CodexProvider {
|
||||
{
|
||||
return Err(continuation_replay_required_error("scope_unavailable"));
|
||||
}
|
||||
// 其他协议必须先取得真实账号,再按固定 attempt 阶段调用转换器;选号前不能
|
||||
// 把未知正文当成 OpenAI wire 解释会话、亲和或传输字段
|
||||
// 跨协议转换必须在选号后执行;选号前只按各自协议读取明确会话身份
|
||||
let upstream = (generate.protocol_payload().protocol() == OPENAI_PROTOCOL)
|
||||
.then(|| encode_generate_request(generate, upstream_model.as_str(), None))
|
||||
.transpose()
|
||||
@@ -726,10 +725,27 @@ impl Provider for CodexProvider {
|
||||
requires_websocket,
|
||||
)
|
||||
});
|
||||
// 内容亲和只交给选号器;中间件复验、轮次关联和原生续接仍只认真实会话
|
||||
let messages_affinity = crate::credential::derive_messages_session_affinity(
|
||||
generate.protocol_payload(),
|
||||
&[],
|
||||
context.client_api_key_ref(),
|
||||
);
|
||||
if context.continuation_attempt() == ContinuationAttempt::RotateForClientReplay
|
||||
&& (selection_session_affinity.is_none()
|
||||
|| context.account_selection_policy().openai_account_affinity()
|
||||
== AccountAffinity::Relaxed)
|
||||
{
|
||||
// 没有可持续的会话绑定就无法保证完整历史使用替代账号,交回原始容量错误
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
}
|
||||
// Messages 身份与内容亲和仅交给选号器,不建立 OpenAI 轮次或原生续接关联
|
||||
let account_affinity = selection_session_affinity
|
||||
.as_ref()
|
||||
.or(cache_preference.as_ref());
|
||||
.or(cache_preference.as_ref())
|
||||
.or(messages_affinity.as_ref());
|
||||
let selection_started_at = Instant::now();
|
||||
let selection = async {
|
||||
self.selector
|
||||
@@ -760,6 +776,11 @@ impl Provider for CodexProvider {
|
||||
};
|
||||
let account_selection_wait_ms =
|
||||
u64::try_from(selection_started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
if context.continuation_attempt() == ContinuationAttempt::RotateForClientReplay {
|
||||
// Core 已确认允许换号;选号器迁移共享绑定后,完整历史仍须由客户端提供
|
||||
// 不将旧账号的 previous_response_id 或增量输入发送给替代账号
|
||||
return Err(continuation_replay_required_error("account_rotated"));
|
||||
}
|
||||
let adapter = context.upstream_adapter_for_account(
|
||||
candidate.provider(),
|
||||
upstream_model,
|
||||
@@ -944,7 +965,9 @@ impl CodexProvider {
|
||||
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny => {
|
||||
PreviousResponseScope::ExternalUnknown
|
||||
}
|
||||
ContinuationAttempt::None => PreviousResponseScope::ExternalUnknown,
|
||||
ContinuationAttempt::None | ContinuationAttempt::RotateForClientReplay => {
|
||||
PreviousResponseScope::ExternalUnknown
|
||||
}
|
||||
};
|
||||
upstream_request.set_previous_response_id(Some(
|
||||
continuation.upstream_response_id().as_str().to_owned(),
|
||||
|
||||
@@ -533,12 +533,13 @@ pub(super) async fn synchronize_passive_quota(
|
||||
quota: &CodexCredentialQuotaService,
|
||||
account: &ProviderAccount,
|
||||
rate_limits: &[ParsedRateLimits],
|
||||
request_used_quota: bool,
|
||||
) {
|
||||
if rate_limits.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Err(error) = quota
|
||||
.synchronize_passive_rate_limits(account, rate_limits)
|
||||
.synchronize_passive_rate_limits_with_usage(account, rate_limits, request_used_quota)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -554,6 +555,7 @@ pub(super) async fn synchronize_passive_quota_headers(
|
||||
account: &ProviderAccount,
|
||||
headers: &[(String, String)],
|
||||
image_endpoint: bool,
|
||||
request_used_quota: bool,
|
||||
) {
|
||||
let parsed = if image_endpoint {
|
||||
gateway_protocol::openai::events::parse_image_rate_limit_headers(headers)
|
||||
@@ -561,7 +563,13 @@ pub(super) async fn synchronize_passive_quota_headers(
|
||||
parse_rate_limit_headers(headers)
|
||||
};
|
||||
if let Some(rate_limits) = parsed {
|
||||
synchronize_passive_quota(quota, account, std::slice::from_ref(&rate_limits)).await;
|
||||
synchronize_passive_quota(
|
||||
quota,
|
||||
account,
|
||||
std::slice::from_ref(&rate_limits),
|
||||
request_used_quota,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -406,73 +406,87 @@ impl OpenAiWarmupTask {
|
||||
impl ScheduledTask for OpenAiWarmupTask {
|
||||
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let policy = self
|
||||
.quota
|
||||
.runtime_policy()
|
||||
.load_warmup_policy()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
WorkerTaskError::safe("OpenAI warmup policy load failed").with_source(error)
|
||||
})?;
|
||||
if !policy.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
use chrono::Timelike as _;
|
||||
let now = chrono::Utc::now();
|
||||
let local_now = self.timezone.local(now);
|
||||
// 回拨产生的第二个相同时刻不能再次执行
|
||||
if self
|
||||
.timezone
|
||||
.resolve_local(local_now.naive_local())
|
||||
.is_none_or(|first| first != now)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let slot = local_now
|
||||
.naive_local()
|
||||
.with_second(0)
|
||||
.and_then(|value| value.with_nanosecond(0))
|
||||
.ok_or_else(|| WorkerTaskError::safe("invalid warmup slot"))?;
|
||||
if !policy.matches_slot(slot) {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(model) = policy.model() else {
|
||||
return Err(WorkerTaskError::safe("OpenAI warmup model is missing"));
|
||||
};
|
||||
if !self
|
||||
.quota
|
||||
.runtime_policy()
|
||||
.claim_warmup_slot(self.timezone, slot)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
WorkerTaskError::safe("warmup slot is unavailable").with_source(source)
|
||||
})?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(%slot, model, "OpenAI account warmup cycle started");
|
||||
let outcome = tokio::select! {
|
||||
let cron_result = self.run_cron_cycle(&context).await;
|
||||
let bucket_result = tokio::select! {
|
||||
() = context.cancellation().cancelled() => return Ok(()),
|
||||
outcome = self.quota.execute_warmup(model) => outcome,
|
||||
result = self.quota.execute_bucket_warmup_at(chrono::Utc::now()) => result,
|
||||
};
|
||||
match outcome {
|
||||
Ok(summary) => {
|
||||
tracing::info!(
|
||||
warmed_up = summary.warmed_up,
|
||||
skipped_active = summary.skipped_active,
|
||||
skipped_exhausted = summary.skipped_exhausted,
|
||||
failed = summary.failed,
|
||||
"OpenAI account warmup cycle completed"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(error = %error, "OpenAI account warmup cycle failed");
|
||||
}
|
||||
if let Err(error) = bucket_result {
|
||||
tracing::warn!(error = %error, "OpenAI quota bucket warmup cycle failed");
|
||||
}
|
||||
Ok(())
|
||||
cron_result
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl OpenAiWarmupTask {
|
||||
async fn run_cron_cycle(&self, context: &WorkerCycleContext) -> Result<(), WorkerTaskError> {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let policy = self
|
||||
.quota
|
||||
.runtime_policy()
|
||||
.load_warmup_policy()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
WorkerTaskError::safe("OpenAI warmup policy load failed").with_source(error)
|
||||
})?;
|
||||
if !policy.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
use chrono::Timelike as _;
|
||||
let now = chrono::Utc::now();
|
||||
let local_now = self.timezone.local(now);
|
||||
// 回拨产生的第二个相同时刻不能再次执行
|
||||
if self
|
||||
.timezone
|
||||
.resolve_local(local_now.naive_local())
|
||||
.is_none_or(|first| first != now)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let slot = local_now
|
||||
.naive_local()
|
||||
.with_second(0)
|
||||
.and_then(|value| value.with_nanosecond(0))
|
||||
.ok_or_else(|| WorkerTaskError::safe("invalid warmup slot"))?;
|
||||
if !policy.matches_slot(slot) {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(model) = policy.model() else {
|
||||
return Err(WorkerTaskError::safe("OpenAI warmup model is missing"));
|
||||
};
|
||||
if !self
|
||||
.quota
|
||||
.runtime_policy()
|
||||
.claim_warmup_slot(self.timezone, slot)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
WorkerTaskError::safe("warmup slot is unavailable").with_source(source)
|
||||
})?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(%slot, model, "OpenAI account warmup cycle started");
|
||||
let outcome = tokio::select! {
|
||||
() = context.cancellation().cancelled() => return Ok(()),
|
||||
outcome = self.quota.execute_warmup(model) => outcome,
|
||||
};
|
||||
match outcome {
|
||||
Ok(summary) => {
|
||||
tracing::info!(
|
||||
warmed_up = summary.warmed_up,
|
||||
skipped_active = summary.skipped_active,
|
||||
skipped_exhausted = summary.skipped_exhausted,
|
||||
failed = summary.failed,
|
||||
"OpenAI account warmup cycle completed"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(error = %error, "OpenAI account warmup cycle failed");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,6 +266,11 @@ impl CodexCanonicalDecoder {
|
||||
std::mem::take(&mut self.timing_signals)
|
||||
}
|
||||
|
||||
/// 使用累计输出事实判断中途失败的请求,避免把前导事件当作额度使用
|
||||
pub(crate) const fn semantic_output_seen(&self) -> bool {
|
||||
self.semantic_output_seen
|
||||
}
|
||||
|
||||
/// 返回本 attempt 的上游响应回显档位,仅供诊断,不作为本地计费依据
|
||||
#[must_use]
|
||||
pub fn response_service_tier(&self) -> Option<&str> {
|
||||
|
||||
@@ -588,6 +588,14 @@ fn classify_upstream_failure(
|
||||
if status == Some(StatusCode::TOO_MANY_REQUESTS) {
|
||||
return CodexFailureCategory::RateLimited;
|
||||
}
|
||||
// 流内 `error` 帧通常不带 HTTP 状态码;结构化 `invalid_request_error` 与
|
||||
// HTTP 400 是同一个上游拒绝,不能因缺少状态码而归为上游不可用
|
||||
if matches!(source, UpstreamFailureSource::SseFailure)
|
||||
&& status.is_none()
|
||||
&& error_type == "invalid_request_error"
|
||||
{
|
||||
return CodexFailureCategory::InvalidRequest;
|
||||
}
|
||||
match status.map(|status| status.as_u16()) {
|
||||
Some(status) => match status {
|
||||
400 | 404 | 409 | 422 => CodexFailureCategory::InvalidRequest,
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
//! CLI 稳定发布来自官方 npm 包及其平台依赖,不与 Desktop 的内嵌版本混用
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt as _;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::provider_ports::{ProviderArtifactProfile, ProviderArtifactProfileCachePort};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use reqwest::{Client, redirect::Policy};
|
||||
use serde::Deserialize;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::selection::{ClientKind, ClientPlatform, ClientRelease, object};
|
||||
use super::{ARTIFACT_PROFILE_CACHE_TTL, CodexWireProfileState, baseline};
|
||||
use super::{
|
||||
ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL, ARTIFACT_PROFILE_CACHE_TTL, CodexWireProfileState,
|
||||
baseline,
|
||||
};
|
||||
|
||||
const ENDPOINT: &str = "https://registry.npmjs.org/@openai%2Fcodex/latest";
|
||||
const MAX_BYTES: usize = 256 * 1024;
|
||||
@@ -67,11 +71,21 @@ fn sequence(version: &str) -> Result<u64, CliReleaseError> {
|
||||
Ok(1 + version.major * 1_000_000 + version.minor * 1_000 + version.patch)
|
||||
}
|
||||
|
||||
pub struct CliReleaseService {
|
||||
/// 官方 npm 发布读取边界
|
||||
pub trait CliReleaseTransport: Send + Sync {
|
||||
fn fetch(&self) -> BoxFuture<'_, Result<String, CliReleaseError>>;
|
||||
}
|
||||
|
||||
struct OfficialCliReleaseTransport {
|
||||
client: Client,
|
||||
}
|
||||
|
||||
pub struct CliReleaseService {
|
||||
transport: Arc<dyn CliReleaseTransport>,
|
||||
provider: ProviderKind,
|
||||
state: CodexWireProfileState,
|
||||
cache: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
cache_writes: Mutex<BTreeMap<String, Instant>>,
|
||||
}
|
||||
|
||||
impl CliReleaseService {
|
||||
@@ -88,12 +102,27 @@ impl CliReleaseService {
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|_| CliReleaseError::Fetch)?;
|
||||
Ok(Self {
|
||||
client,
|
||||
Ok(Self::with_transport(
|
||||
provider,
|
||||
state,
|
||||
cache,
|
||||
})
|
||||
Arc::new(OfficialCliReleaseTransport { client }),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn with_transport(
|
||||
provider: ProviderKind,
|
||||
state: CodexWireProfileState,
|
||||
cache: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
transport: Arc<dyn CliReleaseTransport>,
|
||||
) -> Self {
|
||||
Self {
|
||||
transport,
|
||||
provider,
|
||||
state,
|
||||
cache,
|
||||
cache_writes: Mutex::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn restore(&self) {
|
||||
@@ -135,7 +164,7 @@ impl CliReleaseService {
|
||||
}
|
||||
|
||||
pub async fn refresh(&self) -> Result<(), CliReleaseError> {
|
||||
let result = self.fetch().await;
|
||||
let result = self.transport.fetch().await;
|
||||
for (platform, arch, target) in TARGETS {
|
||||
let outcome = match &result {
|
||||
Ok(version) => self.publish(platform, arch, target, version).await,
|
||||
@@ -155,16 +184,29 @@ impl CliReleaseService {
|
||||
version: &str,
|
||||
) -> Result<ClientRelease, String> {
|
||||
let next = sequence(version).map_err(|error| error.to_string())?;
|
||||
if self
|
||||
.state
|
||||
.client_release(ClientKind::Cli, platform, arch)
|
||||
.is_some_and(|current| {
|
||||
sequence(¤t.codex_version).is_ok_and(|current| current > next)
|
||||
})
|
||||
{
|
||||
let current = self.state.client_release(ClientKind::Cli, platform, arch);
|
||||
if current.as_ref().is_some_and(|current| {
|
||||
sequence(¤t.codex_version).is_ok_and(|current| current > next)
|
||||
}) {
|
||||
return Err("官方 CLI 发布版本低于当前已核验版本".to_owned());
|
||||
}
|
||||
let verified_at = Utc::now();
|
||||
let unchanged = current.filter(|current| current.codex_version == version);
|
||||
if let Some(current) = &unchanged {
|
||||
let fresh = self
|
||||
.cache_writes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(target)
|
||||
.is_some_and(|written| written.elapsed() < ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL);
|
||||
if fresh {
|
||||
return Ok(current.clone());
|
||||
}
|
||||
}
|
||||
// 同版探测和缓存续期保留实际核验时间,只有新版才产生新的核验时间
|
||||
let verified_at = unchanged
|
||||
.as_ref()
|
||||
.and_then(|release| release.verified_at)
|
||||
.unwrap_or_else(Utc::now);
|
||||
let payload =
|
||||
object(&serde_json::json!({"version": version})).map_err(|error| error.to_string())?;
|
||||
let profile = ProviderArtifactProfile::new(
|
||||
@@ -179,41 +221,51 @@ impl CliReleaseService {
|
||||
.replace_if_newer(profile, ARTIFACT_PROFILE_CACHE_TTL)
|
||||
.await
|
||||
{
|
||||
Ok(true) => Ok(ClientRelease {
|
||||
codex_version: version.to_owned(),
|
||||
desktop_version: None,
|
||||
desktop_build: None,
|
||||
verified_at: Some(verified_at),
|
||||
}),
|
||||
Ok(true) => {
|
||||
self.cache_writes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(target.to_owned(), Instant::now());
|
||||
Ok(ClientRelease {
|
||||
codex_version: version.to_owned(),
|
||||
desktop_version: None,
|
||||
desktop_build: None,
|
||||
verified_at: Some(verified_at),
|
||||
})
|
||||
}
|
||||
Ok(false) => Err("CLI 发布缓存已更新,等待下次同步".to_owned()),
|
||||
Err(_) => Err("CLI 发布版本缓存写入失败".to_owned()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch(&self) -> Result<String, CliReleaseError> {
|
||||
let response = self
|
||||
.client
|
||||
.get(ENDPOINT)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| CliReleaseError::Fetch)?;
|
||||
if !response.status().is_success()
|
||||
|| response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > MAX_BYTES as u64)
|
||||
{
|
||||
return Err(CliReleaseError::Fetch);
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|_| CliReleaseError::Fetch)?;
|
||||
if bytes.len().saturating_add(chunk.len()) > MAX_BYTES {
|
||||
return Err(CliReleaseError::Invalid);
|
||||
impl CliReleaseTransport for OfficialCliReleaseTransport {
|
||||
fn fetch(&self) -> BoxFuture<'_, Result<String, CliReleaseError>> {
|
||||
Box::pin(async move {
|
||||
let response = self
|
||||
.client
|
||||
.get(ENDPOINT)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| CliReleaseError::Fetch)?;
|
||||
if !response.status().is_success()
|
||||
|| response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > MAX_BYTES as u64)
|
||||
{
|
||||
return Err(CliReleaseError::Fetch);
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
parse_cli_release(&bytes)
|
||||
let mut bytes = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|_| CliReleaseError::Fetch)?;
|
||||
if bytes.len().saturating_add(chunk.len()) > MAX_BYTES {
|
||||
return Err(CliReleaseError::Invalid);
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
parse_cli_release(&bytes)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,11 +34,12 @@ use selection::{ClientKind, ClientPlatform, ClientRelease};
|
||||
pub const CODEX_DESKTOP_APPCAST_URL: &str =
|
||||
"https://persistent.oaistatic.com/codex-app-prod/appcast.xml";
|
||||
/// 官方发布元数据检查周期
|
||||
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
|
||||
const APPCAST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_APPCAST_BYTES: usize = 1024 * 1024;
|
||||
const ARTIFACT_PROFILE_CACHE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
|
||||
const ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1;
|
||||
|
||||
/// 保留已有配置类型入口;位置规则由出口领域统一维护
|
||||
@@ -634,6 +635,7 @@ pub struct CodexDesktopReleaseService {
|
||||
profile: CodexWireProfileState,
|
||||
cache: CodexArtifactProfileCache,
|
||||
verified: Arc<RwLock<Option<CodexBundledReleaseProfile>>>,
|
||||
cache_written_at: Arc<RwLock<Option<tokio::time::Instant>>>,
|
||||
}
|
||||
|
||||
impl CodexDesktopReleaseService {
|
||||
@@ -668,6 +670,7 @@ impl CodexDesktopReleaseService {
|
||||
profile,
|
||||
cache,
|
||||
verified: Arc::new(RwLock::new(verified)),
|
||||
cache_written_at: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -732,10 +735,24 @@ impl CodexDesktopReleaseService {
|
||||
{
|
||||
return Err(CodexDesktopReleaseError::InconsistentRelease);
|
||||
}
|
||||
if !self.cache.replace_if_newer(&verified).await? {
|
||||
return Err(CodexDesktopReleaseError::ReleaseRollback);
|
||||
// 轻量检查只推进观察时间,缓存单独按天续期
|
||||
let renewal_due = self
|
||||
.cache_written_at
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.is_none_or(|written| {
|
||||
written.elapsed() >= ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL
|
||||
});
|
||||
if renewal_due {
|
||||
if !self.cache.replace_if_newer(&verified).await? {
|
||||
return Err(CodexDesktopReleaseError::ReleaseRollback);
|
||||
}
|
||||
*self
|
||||
.cache_written_at
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||||
Some(tokio::time::Instant::now());
|
||||
}
|
||||
self.profile.update_bundled_release(&verified);
|
||||
return Ok(release);
|
||||
}
|
||||
}
|
||||
@@ -766,6 +783,10 @@ impl CodexDesktopReleaseService {
|
||||
.verified
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(published);
|
||||
*self
|
||||
.cache_written_at
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(tokio::time::Instant::now());
|
||||
Ok(release)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
use super::selection::{
|
||||
ClientKind, ClientPlatform, ClientProfileSelection, ClientRelease, VersionMode, object,
|
||||
};
|
||||
use super::{ARTIFACT_PROFILE_CACHE_TTL, CodexWireProfileState, baseline};
|
||||
use super::{
|
||||
ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL, ARTIFACT_PROFILE_CACHE_TTL, CodexWireProfileState,
|
||||
baseline,
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::provider_ports::{ProviderArtifactProfile, ProviderArtifactProfileCachePort};
|
||||
@@ -135,6 +138,7 @@ pub struct PlatformDesktopReleaseService {
|
||||
cache: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
transport: Arc<dyn DesktopArtifactTransport>,
|
||||
verified: Mutex<BTreeMap<DesktopTarget, VerifiedArtifact>>,
|
||||
cache_written_at: Mutex<BTreeMap<DesktopTarget, tokio::time::Instant>>,
|
||||
}
|
||||
impl PlatformDesktopReleaseService {
|
||||
pub fn new(
|
||||
@@ -149,6 +153,7 @@ impl PlatformDesktopReleaseService {
|
||||
cache,
|
||||
transport,
|
||||
verified: Mutex::new(baseline_artifacts()),
|
||||
cache_written_at: Mutex::default(),
|
||||
}
|
||||
}
|
||||
pub async fn restore(&self) {
|
||||
@@ -206,14 +211,32 @@ impl PlatformDesktopReleaseService {
|
||||
.cloned();
|
||||
let mut artifact = self
|
||||
.transport
|
||||
.fetch(target, previous)
|
||||
.fetch(target, previous.clone())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let sequence = sequence(target, &artifact.release).map_err(|error| error.to_string())?;
|
||||
if !self.accepts(target, &artifact.release) {
|
||||
return Err("官方 Desktop 发布低于当前版本或同构建版本不一致".to_owned());
|
||||
}
|
||||
let verified_at = Utc::now();
|
||||
let unchanged = previous
|
||||
.as_ref()
|
||||
.is_some_and(|previous| previous.identity == artifact.identity);
|
||||
if unchanged
|
||||
&& self
|
||||
.cache_written_at
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&target)
|
||||
.is_some_and(|written| written.elapsed() < ARTIFACT_PROFILE_CACHE_RENEW_INTERVAL)
|
||||
{
|
||||
return Ok(artifact.release);
|
||||
}
|
||||
// 相同制品只续期缓存,保留实际提取版本时的核验时间
|
||||
let verified_at = if unchanged {
|
||||
artifact.release.verified_at.unwrap_or_else(Utc::now)
|
||||
} else {
|
||||
Utc::now()
|
||||
};
|
||||
// 核验时间属于缓存信封,避免同构建重复检查被判定为不同制品
|
||||
artifact.release.verified_at = None;
|
||||
let payload =
|
||||
@@ -235,6 +258,10 @@ impl PlatformDesktopReleaseService {
|
||||
Ok(false) => return Err("Desktop 发布缓存已更新,等待下次同步".to_owned()),
|
||||
Err(_) => return Err("Desktop 发布缓存写入失败".to_owned()),
|
||||
}
|
||||
self.cache_written_at
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(target, tokio::time::Instant::now());
|
||||
artifact.release.verified_at = Some(verified_at);
|
||||
self.verified
|
||||
.lock()
|
||||
|
||||
@@ -282,7 +282,7 @@ impl ClientProfileSelection {
|
||||
verified_at: release.verified_at.unwrap_or(DateTime::UNIX_EPOCH),
|
||||
};
|
||||
if let Some(entry) = self.cli_entry {
|
||||
// 入口后缀使用同一次解析得到的 Core 版本,避免每日更新后头部与后缀混用
|
||||
// 入口后缀使用同一次解析得到的 Core 版本,避免版本更新后头部与后缀混用
|
||||
// originator 可单独覆盖;入口名与官方 clientInfo.name 的语义保持一致
|
||||
profile.exact_user_agent = Some(format!(
|
||||
"{} ({}; {})",
|
||||
|
||||
@@ -320,7 +320,7 @@ async fn dial_account(
|
||||
*proxy_hint = None;
|
||||
let tls = tls::account_proxy_tls_config().map_err(|_| invalid())?;
|
||||
let name = rustls_pki_types::ServerName::try_from(proxy_host).map_err(|_| invalid())?;
|
||||
*proxy_hint = Some(ProxyConnectionHint::Unreachable);
|
||||
*proxy_hint = Some(ProxyConnectionHint::CheckProxy);
|
||||
MaybeTlsStream::Rustls(
|
||||
tokio_rustls::TlsConnector::from(tls)
|
||||
.connect(name, tcp)
|
||||
|
||||
@@ -2185,7 +2185,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
quota_paused: false,
|
||||
quota_limits: Default::default(),
|
||||
quota_settings: Default::default(),
|
||||
auto_use_quota_credits: false,
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user