mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4127886584 | ||
|
|
ee3e5db6c2 | ||
|
|
153cb67b9c | ||
|
|
f370d355b6 | ||
|
|
a31dad992c | ||
|
|
55a88f5f6b | ||
|
|
5d90a92582 | ||
|
|
93abe8efcd | ||
|
|
f4b75c42ed | ||
|
|
36ff038c78 | ||
|
|
545009590c | ||
|
|
5b7b9b21ee | ||
|
|
44a2a87b69 | ||
|
|
184e88083b | ||
|
|
53b40aea46 | ||
|
|
aa328b6dbf | ||
|
|
64231ff403 | ||
|
|
d7b80a2066 | ||
|
|
08d63ea3f3 | ||
|
|
ac9c4c85ad | ||
|
|
492c401599 | ||
|
|
a063258a40 | ||
|
|
dc590f2b5f | ||
|
|
ca52e2bfd9 | ||
|
|
43604e36dc | ||
|
|
a4a997e351 | ||
|
|
73ca463924 | ||
|
|
9eeec2c78c | ||
|
|
90890be2b4 | ||
|
|
2e4bab17a8 | ||
|
|
b052e79a16 | ||
|
|
9775451e83 | ||
|
|
c22a2ad144 | ||
|
|
7bd362d544 | ||
|
|
e7036a616f | ||
|
|
98652d4c68 | ||
|
|
d0dc7ef92b | ||
|
|
194fb20d5f | ||
|
|
30f528f084 | ||
|
|
28d024ceda | ||
|
|
6b9c98c59f | ||
|
|
1294e50b76 | ||
|
|
a0e540642b | ||
|
|
14d6a7de31 | ||
|
|
cd7b60a3ba | ||
|
|
ebbed22f36 | ||
|
|
f6e5d8f673 | ||
|
|
bf1c6211b8 | ||
|
|
d388fa344a | ||
|
|
eabb61b08c | ||
|
|
a5c3438927 | ||
|
|
b6948542bb | ||
|
|
0242bd3193 | ||
|
|
229ef48470 | ||
|
|
19c35ba901 | ||
|
|
3c06ae97e8 | ||
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab | ||
|
|
95f6434443 | ||
|
|
8734b0d3d5 | ||
|
|
790d0d9628 | ||
|
|
3c562563c5 | ||
|
|
f19785d0e1 | ||
|
|
a3d098a87c | ||
|
|
449d7c4218 | ||
|
|
d199141a91 | ||
|
|
df83ffdafb | ||
|
|
cf03c56cfa | ||
|
|
3d1e33558a | ||
|
|
f314ebb3a2 |
@@ -21,6 +21,11 @@ updates:
|
||||
day: monday
|
||||
time: '09:10'
|
||||
timezone: Asia/Shanghai
|
||||
# 固定官方 Codex Desktop 的 TLS 指纹;升级前必须抓取握手并比对扩展顺序。
|
||||
ignore:
|
||||
- dependency-name: hyper-rustls
|
||||
- dependency-name: reqwest
|
||||
- dependency-name: rustls
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
components: rustfmt,clippy
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
- name: Detect affected checks
|
||||
id: filter
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
|
||||
@@ -375,7 +375,7 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
@@ -451,7 +451,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -522,7 +522,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -581,7 +581,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
Generated
+125
-145
@@ -216,9 +216,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.140.0"
|
||||
version = "1.141.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9660cf991e512fbe6094f1041ff3d3282bc5aeeeb6184e8de437ec2de024a10"
|
||||
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -239,7 +239,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http-body 1.0.1",
|
||||
@@ -265,7 +265,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"percent-encoding",
|
||||
@@ -566,9 +566,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "base16ct"
|
||||
version = "0.2.0"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
|
||||
checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
@@ -578,9 +578,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.0"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64-simd"
|
||||
@@ -793,12 +793,6 @@ dependencies = [
|
||||
"yaml-rust2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.2"
|
||||
@@ -850,6 +844,12 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cpubits"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -940,12 +940,16 @@ checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.5.5"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core 0.6.4",
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
"getrandom 0.4.3",
|
||||
"hybrid-array",
|
||||
"num-traits",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -967,7 +971,9 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"hybrid-array",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1007,6 +1013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1075,24 +1082,14 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.7.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
||||
dependencies = [
|
||||
"const-oid 0.9.6",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"const-oid",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -1109,7 +1106,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
@@ -1121,7 +1117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.1",
|
||||
"const-oid 0.10.2",
|
||||
"const-oid",
|
||||
"crypto-common 0.2.2",
|
||||
"ctutils",
|
||||
]
|
||||
@@ -1160,16 +1156,17 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.16.9"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"digest 0.10.7",
|
||||
"der",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature 2.2.0",
|
||||
"spki 0.7.3",
|
||||
"signature 3.0.0",
|
||||
"spki",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1178,7 +1175,7 @@ version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a"
|
||||
dependencies = [
|
||||
"pkcs8 0.11.0",
|
||||
"pkcs8",
|
||||
"signature 3.0.0",
|
||||
]
|
||||
|
||||
@@ -1208,19 +1205,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.13.8"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
|
||||
checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
"digest 0.10.7",
|
||||
"crypto-common 0.2.2",
|
||||
"digest 0.11.3",
|
||||
"ff",
|
||||
"generic-array",
|
||||
"group",
|
||||
"hybrid-array",
|
||||
"pem-rfc7468",
|
||||
"pkcs8 0.10.2",
|
||||
"rand_core 0.6.4",
|
||||
"pkcs8",
|
||||
"rand_core 0.10.1",
|
||||
"sec1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -1289,11 +1287,11 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
|
||||
checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -1495,7 +1493,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"cron",
|
||||
@@ -1521,7 +1519,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -1677,12 +1675,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.13.0"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
|
||||
checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"rand_core 0.6.4",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -1779,16 +1777,7 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"hmac",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1879,7 +1868,9 @@ version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
|
||||
dependencies = [
|
||||
"subtle",
|
||||
"typenum",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2134,11 +2125,9 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"js-sys",
|
||||
"pem",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signature 2.2.0",
|
||||
"simple_asn1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -2289,16 +2278,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
@@ -2359,14 +2338,15 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
version = "0.13.2"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b"
|
||||
checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a"
|
||||
dependencies = [
|
||||
"ecdsa",
|
||||
"elliptic-curve",
|
||||
"primefield",
|
||||
"primeorder",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2415,21 +2395,11 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "0.7.0"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
|
||||
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
]
|
||||
@@ -2512,24 +2482,14 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
"der",
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2574,12 +2534,30 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primeorder"
|
||||
version = "0.13.6"
|
||||
name = "primefield"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6"
|
||||
checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4"
|
||||
dependencies = [
|
||||
"crypto-bigint",
|
||||
"crypto-common 0.2.2",
|
||||
"ff",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primeorder"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06"
|
||||
dependencies = [
|
||||
"elliptic-curve",
|
||||
"once_cell",
|
||||
"primefield",
|
||||
"serdect",
|
||||
"wnaf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2597,7 +2575,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"cookie",
|
||||
@@ -2610,7 +2588,6 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"hex",
|
||||
"hyper-rustls",
|
||||
"jsonwebtoken",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"rustls",
|
||||
@@ -2628,6 +2605,7 @@ dependencies = [
|
||||
"tokio-tungstenite 0.28.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tungstenite 0.27.0",
|
||||
"url",
|
||||
"uuid",
|
||||
@@ -2641,7 +2619,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -2833,9 +2811,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.4.1"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a"
|
||||
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -2948,12 +2926,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.4.0"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2"
|
||||
checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b"
|
||||
dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"subtle",
|
||||
"crypto-bigint",
|
||||
"hmac",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3095,14 +3073,14 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "sec1"
|
||||
version = "0.7.3"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der 0.7.10",
|
||||
"generic-array",
|
||||
"pkcs8 0.10.2",
|
||||
"ctutils",
|
||||
"der",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -3213,6 +3191,16 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serdect"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.7"
|
||||
@@ -3294,7 +3282,6 @@ version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
@@ -3303,6 +3290,10 @@ name = "signature"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5"
|
||||
dependencies = [
|
||||
"digest 0.11.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
@@ -3310,18 +3301,6 @@ version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||
|
||||
[[package]]
|
||||
name = "simple_asn1"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"thiserror",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.3"
|
||||
@@ -3368,16 +3347,6 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.7.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.8.0"
|
||||
@@ -3385,7 +3354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.1",
|
||||
"der",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3523,7 +3492,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hkdf",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
@@ -3654,7 +3623,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -4493,6 +4462,17 @@ version = "0.57.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
||||
|
||||
[[package]]
|
||||
name = "wnaf"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"group",
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.3"
|
||||
|
||||
@@ -134,7 +134,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_core_version() {
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
|
||||
}
|
||||
|
||||
|
||||
@@ -220,7 +220,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -229,9 +230,12 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -394,7 +398,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -403,10 +408,13 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -448,8 +456,12 @@ impl BackupTask {
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "上传到对象存储失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -475,19 +487,46 @@ impl BackupTask {
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(Some(remote)) => {
|
||||
let size_matches = remote.size_bytes == size_bytes;
|
||||
let sha256_matches = remote.sha256 == sha256;
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
expected_size_bytes = size_bytes,
|
||||
remote_size_bytes = remote.size_bytes,
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"远端对象大小或校验值不一致",
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"上传完成后未找到远端对象",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "远端对象校验失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -500,10 +539,17 @@ impl BackupTask {
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let message = message.into();
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
error_code,
|
||||
error_message = %message,
|
||||
"备份任务失败"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
error_code: Some(error_code.to_string()),
|
||||
error_message: Some(message.into()),
|
||||
error_message: Some(message),
|
||||
completed_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -13,13 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
Expired,
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -65,7 +198,7 @@ pub struct AccountRecord {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -75,9 +208,9 @@ pub struct AccountRecord {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -170,13 +303,15 @@ pub struct AccountPage {
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
|
||||
@@ -443,14 +443,17 @@ pub struct RequestMetricPoint {
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted` 仍单列,以便状态详情展示和诊断。
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
@@ -528,12 +531,23 @@ pub struct DashboardRuntimeSlots {
|
||||
pub used_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的公共观测事实。
|
||||
/// 仪表盘卡片脚注展示的全历史累计。
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的当前区间事实及全历史累计。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: AccountPoolMetrics,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub account_usage: Vec<DashboardAccountUsage>,
|
||||
|
||||
@@ -11,7 +11,10 @@ use gateway_core::{
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -143,7 +146,7 @@ pub struct PreparedCredentialCreate {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -153,8 +156,6 @@ pub struct PreparedCredentialCreate {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -563,6 +564,15 @@ impl fmt::Debug for RotateCredential {
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用账号用量能否可靠归属到一个 Provider quota 窗口。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum QuotaLocalUsageAttribution {
|
||||
/// 窗口覆盖账号的全部请求,可按账号与时间范围聚合。
|
||||
AccountWide,
|
||||
/// 窗口需要 Provider / 模型级归属,通用账号聚合不可用。
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
/// 一个 Provider quota 窗口的公共投影。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderQuotaWindow {
|
||||
@@ -570,9 +580,11 @@ pub struct ProviderQuotaWindow {
|
||||
pub group: String,
|
||||
pub label: String,
|
||||
pub source: Option<String>,
|
||||
pub local_usage_attribution: QuotaLocalUsageAttribution,
|
||||
pub window_seconds: Option<u64>,
|
||||
pub used_percent: Option<f64>,
|
||||
pub reset_at: Option<DateTime<Utc>>,
|
||||
pub limit_reached: bool,
|
||||
pub local_usage: Option<AccountUsage>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
@@ -591,6 +603,11 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
@@ -687,6 +704,8 @@ impl fmt::Debug for ProviderExport {
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,13 +18,14 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
PrepareCredentialRefresh, ProviderModels, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderQuotaWindow,
|
||||
ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -159,7 +160,9 @@ impl DefaultAccountsService {
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (account, quota) in accounts.iter().zip(quotas) {
|
||||
for window in &mut quota.windows {
|
||||
if window.local_usage.is_none() {
|
||||
if window.local_usage.is_none()
|
||||
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
let key = (account.id.clone(), window.key.clone());
|
||||
if let Some(usage) = usage_by_window.get(&key) {
|
||||
window.local_usage = Some(usage.clone());
|
||||
@@ -216,8 +219,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -260,18 +266,37 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
provider
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
// 该账号降级为空额度投影,其余账号与页面状态照常返回。
|
||||
let provider = match self.providers.require(&account.provider_kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Ok(empty_quota());
|
||||
}
|
||||
};
|
||||
match provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: rolling_usage.get(&account.id).cloned(),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))
|
||||
{
|
||||
Ok(quota) => Ok(quota),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
Ok(empty_quota())
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -282,11 +307,16 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -407,7 +437,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, Utc::now());
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -441,7 +471,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, Utc::now())
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
@@ -486,31 +516,48 @@ impl AccountsService for DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
AccountStatus::Active
|
||||
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
@@ -523,7 +570,9 @@ fn quota_usage_window(
|
||||
account_id: &str,
|
||||
window: &ProviderQuotaWindow,
|
||||
) -> Option<AccountUsageWindowQuery> {
|
||||
if window.local_usage.is_some() {
|
||||
if window.local_usage.is_some()
|
||||
|| window.local_usage_attribution != QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let reset_at = window.reset_at?;
|
||||
|
||||
@@ -112,15 +112,10 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let today =
|
||||
dashboard_period_metrics(&observation.trend, today_start, observation.range.end);
|
||||
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
|
||||
let total_billing_usd = observation
|
||||
.attempts
|
||||
.costs
|
||||
.iter()
|
||||
.find(|cost| cost.currency == "USD")
|
||||
.map(|cost| cost.amount.clone());
|
||||
let total_billing_usd = observation.totals.billing_usd.clone();
|
||||
let total_cached_token_rate = rate_or_zero(
|
||||
observation.requests.cached_tokens,
|
||||
observation.requests.input_tokens,
|
||||
observation.totals.cached_tokens,
|
||||
observation.totals.input_tokens,
|
||||
);
|
||||
let average_first_token_latency_ms = average(
|
||||
observation.requests.first_token_latency_sum_ms,
|
||||
|
||||
@@ -84,25 +84,29 @@ impl DefaultOpenAiService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
let account_ids = account_ids.to_vec();
|
||||
let request_id = request_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,8 +159,7 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
|
||||
.await;
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -208,8 +211,7 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
|
||||
@@ -13,9 +13,10 @@ use gateway_admin::{
|
||||
MutationActor, MutationContext, Revision,
|
||||
auth::AdminAuditEvent,
|
||||
backup::{
|
||||
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
|
||||
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
|
||||
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
|
||||
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
|
||||
UpdateBackupStorageCommand,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -497,15 +498,23 @@ impl DatabaseDumpPort for FakeDumpPort {
|
||||
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让上传返回指定的脱敏 S3 错误。
|
||||
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
|
||||
self.upload_error = Some(BackupError::new(code, message.into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
@@ -530,6 +539,9 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
_config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.upload_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let content = std::fs::read(&request.source).map_err(|_| {
|
||||
gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
|
||||
|
||||
@@ -6,9 +6,9 @@ use chrono::{Duration, Utc};
|
||||
use gateway_admin::{
|
||||
backup::task::BackupTask,
|
||||
model::backup::{
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
|
||||
},
|
||||
ports::backup::BackupRepository,
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
|
||||
@@ -89,6 +89,78 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
assert!(object_store.object(&seed.object_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("recovered-upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
dump.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("stage dump");
|
||||
repository.force_status(&seed.id, BackupStatus::Dumping);
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&cancellation).await.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_cleans_expired_scheduled_backups() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
|
||||
@@ -11,6 +11,7 @@ use gateway_admin::{
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::AdminApiKey,
|
||||
},
|
||||
@@ -127,6 +128,8 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(18_000), Some(30.0)),
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
@@ -143,9 +146,11 @@ fn quota_window(
|
||||
group: group.to_owned(),
|
||||
label: group.to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ use gateway_admin::{
|
||||
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -373,7 +373,7 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
accounts: Mutex<Vec<AccountRecord>>,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
@@ -388,7 +388,7 @@ impl FakeAccountStore {
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
@@ -408,6 +408,10 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
@@ -439,28 +443,41 @@ impl FakeAccountStore {
|
||||
#[async_trait]
|
||||
impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: vec![self.account.clone()],
|
||||
total: 1,
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
summary: AccountSummary {
|
||||
total: 1,
|
||||
active: 1,
|
||||
total: accounts.len() as u64,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.account.clone());
|
||||
Ok((account_id == account.id.as_str()).then_some(account))
|
||||
.filter(|account| account.id == account_id)
|
||||
.or_else(|| {
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -488,11 +505,12 @@ impl AccountStore for FakeAccountStore {
|
||||
_: CredentialListQuery,
|
||||
) -> AdminStoreResult<CredentialPage> {
|
||||
self.record("store.list_credentials");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(CredentialPage {
|
||||
config_revision: revision(1),
|
||||
items: (provider_kind == &self.account.provider_kind)
|
||||
.then(|| self.account.clone())
|
||||
items: accounts
|
||||
.into_iter()
|
||||
.filter(|account| &account.provider_kind == provider_kind)
|
||||
.collect(),
|
||||
next_cursor: None,
|
||||
})
|
||||
@@ -504,14 +522,19 @@ impl AccountStore for FakeAccountStore {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> AdminStoreResult<Option<CredentialDetails>> {
|
||||
self.record("store.credential_details");
|
||||
Ok(
|
||||
(provider_kind == &self.account.provider_kind
|
||||
&& account_id.as_str() == self.account.id)
|
||||
.then(|| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential: self.account.clone(),
|
||||
}),
|
||||
)
|
||||
let account = self
|
||||
.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| {
|
||||
&account.provider_kind == provider_kind && account.id == account_id.as_str()
|
||||
})
|
||||
.cloned();
|
||||
Ok(account.map(|credential| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn load_credentials_for_export(
|
||||
@@ -520,21 +543,26 @@ impl AccountStore for FakeAccountStore {
|
||||
account_ids: &[ProviderAccountId],
|
||||
) -> AdminStoreResult<Vec<ProviderExportCredentialInput>> {
|
||||
self.record("store.load_credentials_for_export");
|
||||
if provider_kind != &self.account.provider_kind
|
||||
|| account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() != self.account.id)
|
||||
{
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
if accounts.iter().any(|account| {
|
||||
&account.provider_kind != provider_kind
|
||||
|| !account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() == account.id)
|
||||
}) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::NotFound,
|
||||
"test account",
|
||||
"credential not found",
|
||||
));
|
||||
}
|
||||
Ok(vec![ProviderExportCredentialInput {
|
||||
account: self.account.clone(),
|
||||
provider_material: document(),
|
||||
}])
|
||||
Ok(accounts
|
||||
.into_iter()
|
||||
.map(|account| ProviderExportCredentialInput {
|
||||
account,
|
||||
provider_material: document(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn commit_credential_import(
|
||||
@@ -818,7 +846,7 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
@@ -841,7 +869,7 @@ async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
..
|
||||
})
|
||||
));
|
||||
@@ -880,7 +908,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let expected = store.account.clone();
|
||||
let expected = store.accounts.lock().expect("accounts")[0].clone();
|
||||
accounts_service(provider.clone(), store)
|
||||
.await
|
||||
.accounts()
|
||||
@@ -922,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -958,22 +986,88 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() {
|
||||
let events = events();
|
||||
let openai = FakeProviderAdmin::new("openai", events.clone());
|
||||
openai.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_bad_xai".to_owned();
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(openai)
|
||||
.provider(failing)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("one failing quota must not fail the directory page");
|
||||
|
||||
assert_eq!(page.items.len(), 2);
|
||||
let by_id = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| (item.account.id.as_str(), item))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
let healthy = by_id.get("acct_test").expect("healthy account item");
|
||||
assert_eq!(healthy.quota.windows.len(), 1);
|
||||
assert_eq!(healthy.quota.windows[0].key, "primary");
|
||||
let degraded = by_id.get("acct_bad_xai").expect("degraded account item");
|
||||
assert!(degraded.quota.windows.is_empty());
|
||||
assert!(degraded.quota.observed_at.is_none());
|
||||
assert!(degraded.quota.provider_data.is_none());
|
||||
assert_eq!(page.summary.total, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
@@ -997,6 +1091,127 @@ async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_exp
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1009,12 +1224,16 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1045,6 +1264,96 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
assert_eq!(usage.total_tokens, Some(4_330_000));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let reset_at = Utc::now() + TimeDelta::days(7);
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![
|
||||
ProviderQuotaWindow {
|
||||
key: "core-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "周额度".to_owned(),
|
||||
source: Some("codex".to_owned()),
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(1.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
ProviderQuotaWindow {
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "GPT-5.3-Codex-Spark · 周额度".to_owned(),
|
||||
source: Some("codex_bengalfox".to_owned()),
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(0.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
store.set_quota_window_usage(vec![
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "core-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 12_818_806),
|
||||
},
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 127_926),
|
||||
},
|
||||
]);
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota window usage");
|
||||
|
||||
let local_tokens = page.items[0]
|
||||
.quota
|
||||
.windows
|
||||
.iter()
|
||||
.map(|window| {
|
||||
(
|
||||
window.key.as_str(),
|
||||
window
|
||||
.local_usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.total_tokens),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
local_tokens,
|
||||
[
|
||||
("core-primary", Some(12_818_806)),
|
||||
("codex-bengalfox-primary", None),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
let events = events();
|
||||
@@ -1122,6 +1431,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1156,7 +1467,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
provider_kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
name: "test account".to_owned(),
|
||||
email: Some("test@example.invalid".to_owned()),
|
||||
upstream_user_id: "upstream-user".to_owned(),
|
||||
upstream_user_id: Some("upstream-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1166,9 +1477,8 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -1194,7 +1504,7 @@ fn prepared_create_with_id(
|
||||
provider_kind,
|
||||
name: name.to_owned(),
|
||||
email: Some("prepared@example.invalid".to_owned()),
|
||||
upstream_user_id: "prepared-user".to_owned(),
|
||||
upstream_user_id: Some("prepared-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1204,8 +1514,6 @@ fn prepared_create_with_id(
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
}
|
||||
}
|
||||
@@ -1290,9 +1598,7 @@ impl AccountProbe for FreeModelQuotaProbe {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24));
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
|
||||
@@ -449,6 +449,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
range,
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
account_usage: Vec::new(),
|
||||
|
||||
@@ -125,6 +125,8 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
.await
|
||||
.expect("import credential");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -162,6 +164,8 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.await
|
||||
.expect("import credentials");
|
||||
|
||||
await_quota_requests(&provider, 2).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[
|
||||
@@ -203,6 +207,8 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.await
|
||||
.expect("committed import remains successful");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[ProviderAccountId::new("acct_prepared").expect("account ID")]
|
||||
@@ -236,6 +242,8 @@ async fn openai_authorization_create_should_observe_initial_quota() {
|
||||
.await
|
||||
.expect("complete authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -284,6 +292,8 @@ async fn openai_authorization_create_should_remain_successful_when_initial_quota
|
||||
.await
|
||||
.expect("quota observation must not fail authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.account_id.as_str(), "acct_prepared");
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
@@ -322,6 +332,8 @@ async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
.await
|
||||
.expect("released claim must permit retry");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -400,6 +412,8 @@ async fn openai_reauthorization_should_commit_after_credential_revision_advances
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.credential_revision, Some(revision(3)));
|
||||
|
||||
assert_eq!(
|
||||
@@ -426,6 +440,16 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn await_quota_requests(provider: &FakeProviderAdmin, expected: usize) {
|
||||
for _ in 0..100 {
|
||||
if provider.quota_requests().len() == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(provider.quota_requests().len(), expected);
|
||||
}
|
||||
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-openai"),
|
||||
|
||||
@@ -91,7 +91,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -127,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -172,9 +162,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -194,7 +186,10 @@ pub struct AccountView {
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
pub token_refreshing: bool,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
@@ -214,6 +209,9 @@ pub struct AccountView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -227,6 +225,7 @@ pub struct AccountQuotaWindowView {
|
||||
pub label_display: String,
|
||||
pub used_percent: Option<f64>,
|
||||
pub used_percent_display: String,
|
||||
pub limit_reached: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_usage: Option<serde_json::Value>,
|
||||
pub reset_at_display: String,
|
||||
@@ -526,7 +525,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -544,7 +543,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -1332,9 +1331,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1365,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = status.as_str().to_owned();
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -1380,14 +1382,15 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
resource_ref: account.id,
|
||||
email: account.email,
|
||||
account_id: account.upstream_account_id,
|
||||
user_id: Some(account.upstream_user_id),
|
||||
user_id: account.upstream_user_id,
|
||||
label: None,
|
||||
plan_type: account.plan_type,
|
||||
authentication_kind: account.authentication_kind,
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
token_refreshing: false,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
access_token_expires_at: expires_at,
|
||||
@@ -1417,9 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1432,9 +1438,11 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
group,
|
||||
label,
|
||||
source: _,
|
||||
local_usage_attribution: _,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
local_usage,
|
||||
provider_data: _,
|
||||
} = window;
|
||||
@@ -1446,6 +1454,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
used_percent,
|
||||
used_percent_display: used_percent
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
|
||||
limit_reached,
|
||||
local_usage: local_usage.as_ref().map(quota_local_usage),
|
||||
reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)),
|
||||
}
|
||||
@@ -1710,16 +1719,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
|
||||
@@ -709,7 +709,7 @@ pub(crate) fn dashboard_view(
|
||||
} = result;
|
||||
let domain::DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
totals,
|
||||
provider_accounts,
|
||||
account_usage,
|
||||
recent_requests,
|
||||
@@ -756,13 +756,13 @@ pub(crate) fn dashboard_view(
|
||||
today_requests: format_compact_number(today.request_count),
|
||||
today_requests_value: today.request_count,
|
||||
yesterday_requests_value: yesterday.request_count,
|
||||
total_requests: format_compact_number(requests.request_count),
|
||||
total_requests: format_compact_number(totals.request_count),
|
||||
},
|
||||
tokens: DashboardTokensCardView {
|
||||
today_tokens: format_compact_number(today.total_tokens),
|
||||
today_tokens_value: today.total_tokens,
|
||||
yesterday_tokens_value: yesterday.total_tokens,
|
||||
total_tokens: format_compact_number(requests.total_tokens),
|
||||
total_tokens: format_compact_number(totals.total_tokens),
|
||||
total_billing_amount_usd: total_billing_usd
|
||||
.as_ref()
|
||||
.map_or_else(|| "—".to_owned(), |amount| format!("${}", amount.as_str())),
|
||||
@@ -772,7 +772,7 @@ pub(crate) fn dashboard_view(
|
||||
today_hit_rate_value: today.observed_cached_token_rate,
|
||||
yesterday_hit_rate_value: yesterday.observed_cached_token_rate,
|
||||
total_hit_rate: display_rate(total_cached_token_rate),
|
||||
total_cached_tokens: format_compact_number(requests.cached_tokens),
|
||||
total_cached_tokens: format_compact_number(totals.cached_tokens),
|
||||
average_first_token_latency_ms: display_duration(average_first_token_latency_ms),
|
||||
},
|
||||
},
|
||||
@@ -784,7 +784,9 @@ pub(crate) fn dashboard_view(
|
||||
pool_summary: DashboardPoolSummaryView {
|
||||
total: provider_accounts.total,
|
||||
active: provider_accounts.active,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
expired: provider_accounts.expired,
|
||||
invalid: provider_accounts.invalid,
|
||||
quota_exhausted: provider_accounts.quota_exhausted,
|
||||
refreshing: provider_accounts.refreshing,
|
||||
disabled: provider_accounts.disabled,
|
||||
|
||||
@@ -310,16 +310,6 @@ pub(crate) fn usage_range(
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start =
|
||||
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_today_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
|
||||
@@ -38,7 +38,8 @@ where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
// 概览默认按中国时区当日统计,与单独趋势接口保持同一口径。
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
|
||||
@@ -348,7 +348,9 @@ pub struct DashboardAccountRequestBucketView {
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -331,7 +344,7 @@ mod actions {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
@@ -365,7 +378,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
|
||||
@@ -83,6 +83,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -100,12 +101,14 @@ impl AdminTestFixture {
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let dashboard_summary_range = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
@@ -142,6 +145,7 @@ impl AdminTestFixture {
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
dashboard_summary_range,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,6 +417,7 @@ struct UnusedStore {
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -523,7 +528,11 @@ impl AccountStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
*self
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range") = Some(range);
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
|
||||
@@ -219,12 +219,15 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
@@ -268,6 +271,143 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, DashboardTotals, RequestMetrics,
|
||||
TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let range_requests = RequestMetrics {
|
||||
request_count: 2,
|
||||
input_tokens: 8,
|
||||
cached_tokens: 2,
|
||||
total_tokens: 12,
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: range_requests,
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: DashboardTotals {
|
||||
request_count: 42,
|
||||
input_tokens: 800,
|
||||
cached_tokens: 200,
|
||||
total_tokens: 840,
|
||||
billing_usd: None,
|
||||
},
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_lifetime_totals")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["traffic"]["totalRequests"],
|
||||
&value["data"]["cards"]["tokens"]["totalTokens"],
|
||||
&value["data"]["cards"]["cache"]["totalHitRate"],
|
||||
&value["data"]["cards"]["cache"]["totalCachedTokens"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("42"),
|
||||
&serde_json::json!("840"),
|
||||
&serde_json::json!("25.0%"),
|
||||
&serde_json::json!("200"),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_default_to_current_china_day() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
china_day_start,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_today_range")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let range = fixture
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range")
|
||||
.take()
|
||||
.expect("recorded dashboard summary range");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(range.start, china_day_start(range.end));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
use axum::{
|
||||
|
||||
@@ -660,6 +660,9 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -1096,6 +1099,41 @@ where
|
||||
true
|
||||
}
|
||||
|
||||
/// 原生续写的原账号在本地调度阶段已不可用时,改用完整输入重放。
|
||||
///
|
||||
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
|
||||
/// 对应 Provider 的可携带会话状态时,才可安全移除 native handle 并换号。
|
||||
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt != ContinuationAttempt::Native
|
||||
|| !matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(pin) = self
|
||||
.continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if self
|
||||
.operation
|
||||
.provider_session_state(pin.provider().as_str())
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn reset_uncommitted_observations(&mut self) {
|
||||
self.usage = Usage::new();
|
||||
self.cost = CostEstimate::unavailable();
|
||||
|
||||
@@ -188,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData {
|
||||
}
|
||||
|
||||
/// 数据库中固定的账号运行状态。
|
||||
///
|
||||
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
|
||||
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
@@ -206,7 +208,6 @@ impl AccountAvailability {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Ready => "ready",
|
||||
Self::Cooldown => "cooldown",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Expired => "expired",
|
||||
Self::Banned => "banned",
|
||||
@@ -214,13 +215,26 @@ impl AccountAvailability {
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。
|
||||
///
|
||||
/// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用
|
||||
/// 上一次无关的错误信息。
|
||||
#[must_use]
|
||||
pub const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Expired => Some("credential_expired"),
|
||||
Self::Banned => Some("account_banned"),
|
||||
Self::Invalid => Some("credential_invalid"),
|
||||
Self::Unknown | Self::Ready | Self::QuotaExhausted => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析数据库稳定值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"ready" => Some(Self::Ready),
|
||||
"cooldown" => Some(Self::Cooldown),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"expired" => Some(Self::Expired),
|
||||
"banned" => Some(Self::Banned),
|
||||
@@ -237,14 +251,13 @@ pub struct ProviderAccount {
|
||||
provider: ProviderKind,
|
||||
name: String,
|
||||
email: Option<String>,
|
||||
upstream_user_id: String,
|
||||
upstream_user_id: Option<String>,
|
||||
upstream_account_id: Option<String>,
|
||||
plan_type: Option<String>,
|
||||
authentication_kind: String,
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
@@ -257,7 +270,7 @@ impl ProviderAccount {
|
||||
id: ProviderAccountId,
|
||||
provider: ProviderKind,
|
||||
name: String,
|
||||
upstream_user_id: String,
|
||||
upstream_user_id: Option<String>,
|
||||
authentication_kind: String,
|
||||
revision: CredentialRevision,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
@@ -274,7 +287,6 @@ impl ProviderAccount {
|
||||
revision,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Unknown,
|
||||
cooldown_until: None,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
@@ -299,14 +311,20 @@ impl ProviderAccount {
|
||||
mut self,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.availability = availability;
|
||||
self.cooldown_until = cooldown_until;
|
||||
self.availability = if self.upstream_user_id.is_some() {
|
||||
availability
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
};
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 RT 存在性与失败后的最早重试时刻。
|
||||
///
|
||||
/// 正常 OAuth 预刷新由 worker 使用 AT 原始过期时间与当前运行时策略动态判断,
|
||||
/// 不应把提前量物化到 `next_refresh_at`。
|
||||
#[must_use]
|
||||
pub const fn with_refresh_schedule(
|
||||
mut self,
|
||||
@@ -344,8 +362,8 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
pub fn upstream_user_id(&self) -> Option<&str> {
|
||||
self.upstream_user_id.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -373,16 +391,12 @@ impl ProviderAccount {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cooldown_until(&self) -> Option<SystemTime> {
|
||||
self.cooldown_until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn access_token_expires_at(&self) -> Option<SystemTime> {
|
||||
self.access_token_expires_at
|
||||
}
|
||||
|
||||
/// 返回瞬态 OAuth 刷新失败后的最早重试时刻;正常账号为 `None`。
|
||||
#[must_use]
|
||||
pub const fn next_refresh_at(&self) -> Option<SystemTime> {
|
||||
self.next_refresh_at
|
||||
@@ -398,14 +412,14 @@ impl ProviderAccount {
|
||||
pub fn is_schedulable(&self, now: SystemTime) -> bool {
|
||||
let available = match self.availability {
|
||||
AccountAvailability::Ready => true,
|
||||
AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => false,
|
||||
};
|
||||
self.enabled
|
||||
self.upstream_user_id.is_some()
|
||||
&& self.enabled
|
||||
&& available
|
||||
&& self
|
||||
.access_token_expires_at
|
||||
@@ -588,6 +602,8 @@ pub struct QuotaObservation {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: Option<OpaqueProviderData>,
|
||||
pub observed_at: Option<SystemTime>,
|
||||
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
|
||||
pub limit_reached: Option<bool>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for QuotaObservation {
|
||||
@@ -614,9 +630,9 @@ pub struct AccountStateChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub reason: Option<String>,
|
||||
pub cooldown_until: Option<SystemTime>,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口。
|
||||
@@ -756,6 +772,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub quota_limit_reached: bool,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -903,6 +920,7 @@ impl AccountRuntimeSignals {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
self.quota_limit_reached = quota.limit_reached;
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -924,14 +942,20 @@ impl AccountRuntimeSignals {
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl AccountQuotaSignals {
|
||||
#[must_use]
|
||||
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
|
||||
pub const fn new(
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -944,6 +968,12 @@ impl AccountQuotaSignals {
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
|
||||
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实与可重建运行信号的请求级组合。
|
||||
|
||||
@@ -635,6 +635,13 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
/// 生命周期与 credential revision 无关;不要求调用方持有 revision。
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -667,22 +674,20 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按配置的精确提前量计算刷新时刻。
|
||||
pub fn next_attempt_at(
|
||||
/// 判断 AT 是否已经进入当前配置的提前刷新窗口。
|
||||
///
|
||||
/// `margin` 是运行时策略,不投影为账号的持久化时间字段;已过期 AT 也需要
|
||||
/// 尝试用 RT 恢复,因此视为到期。
|
||||
#[must_use]
|
||||
pub fn is_refresh_due(
|
||||
self,
|
||||
_account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
if self.margin >= remaining {
|
||||
return Ok(observed_at);
|
||||
) -> bool {
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= self.margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
access_token_expires_at
|
||||
.checked_sub(self.margin)
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -513,17 +513,27 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
|
||||
}
|
||||
|
||||
fn plan(operation: &Operation) -> RoutingPlan {
|
||||
plan_with_policy(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
account_selection_policy,
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -1514,6 +1524,70 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_native_continuation_replays_with_the_configured_account_policy() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let policy = AccountSelectionPolicy::new(
|
||||
RotationStrategy::RoundRobin,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
);
|
||||
let route_plan = plan_with_policy(&operation, policy);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let original = ProviderAccountId::new("acct_one").expect("account");
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.clone(),
|
||||
);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].account_selection_policy().strategy(),
|
||||
RotationStrategy::RoundRobin
|
||||
);
|
||||
assert!(contexts[1].excluded_accounts().contains(&original));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
|
||||
@@ -19,12 +19,12 @@ fn account(id: &str) -> ProviderAccount {
|
||||
ProviderAccountId::new(id).expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
id.to_owned(),
|
||||
format!("user-{id}"),
|
||||
Some(format!("user-{id}")),
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
|
||||
@@ -35,6 +35,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -110,31 +111,27 @@ fn availability_should_round_trip_all_database_values() {
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_exhausted")
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_disabled")
|
||||
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -158,31 +155,31 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account =
|
||||
account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready, None);
|
||||
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
|
||||
|
||||
assert!(!account.is_schedulable(SystemTime::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_become_schedulable_at_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account =
|
||||
account("acct_cooldown").with_runtime_state(true, AccountAvailability::Cooldown, Some(now));
|
||||
fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
let account = ProviderAccount::new(
|
||||
ProviderAccountId::new("acct_pending_identity").expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
"pending identity".to_owned(),
|
||||
None,
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready);
|
||||
|
||||
assert!(account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_remain_isolated_before_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account = account("acct_cooldown").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::Cooldown,
|
||||
Some(now + Duration::from_secs(1)),
|
||||
assert_eq!(
|
||||
(
|
||||
account.availability(),
|
||||
account.is_schedulable(SystemTime::now())
|
||||
),
|
||||
(AccountAvailability::Unknown, false)
|
||||
);
|
||||
|
||||
assert!(!account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -311,10 +308,15 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(
|
||||
Some(reset_at),
|
||||
Some(75),
|
||||
false,
|
||||
)));
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -629,3 +631,48 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
// availability 终态(Banned/Expired/Invalid)始终排除,
|
||||
// 不受 quota 信号或 skip_exhausted 放宽影响。
|
||||
let mut banned = candidate("acct_banned", 0, Some(100));
|
||||
banned.account = banned
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Banned);
|
||||
let mut expired = candidate("acct_expired", 0, Some(100));
|
||||
expired.account = expired
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Expired);
|
||||
let mut invalid = candidate("acct_invalid", 0, Some(100));
|
||||
invalid.account = invalid
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Invalid);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![banned, expired, invalid, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
|
||||
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
|
||||
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
|
||||
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![exhausted, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
@@ -57,8 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let account = ProviderAccountId::new("acct_stable").expect("valid account");
|
||||
fn refresh_policy_should_mark_tokens_due_at_the_exact_configured_margin() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
NonZeroU32::new(2).expect("positive concurrency"),
|
||||
@@ -67,15 +66,12 @@ fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
let expires_at = observed_at + Duration::from_secs(7_200);
|
||||
|
||||
let first = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh");
|
||||
assert_eq!(first, expires_at - Duration::from_secs(3_600));
|
||||
assert!(!policy.is_refresh_due(expires_at, observed_at));
|
||||
assert!(policy.is_refresh_due(observed_at + Duration::from_secs(3_600), observed_at));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
|
||||
let account = ProviderAccountId::new("acct_boundary").expect("valid account");
|
||||
fn refresh_policy_should_mark_expired_tokens_due() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
@@ -83,12 +79,7 @@ fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
|
||||
.expect("valid policy");
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
|
||||
assert_eq!(
|
||||
policy
|
||||
.next_attempt_at(&account, observed_at + Duration::from_secs(1), observed_at,)
|
||||
.expect("schedule immediate refresh"),
|
||||
observed_at
|
||||
);
|
||||
assert!(policy.is_refresh_due(observed_at - Duration::from_secs(1), observed_at));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -101,6 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -13,6 +13,9 @@ use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::Subscriber
|
||||
use crate::config::LoggingConfig;
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
|
||||
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
@@ -34,7 +37,8 @@ pub enum LogError {
|
||||
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = EnvFilter::try_new(&directive).map_err(|_| LogError::InvalidFilter)?;
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
@@ -47,25 +51,37 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
guards.push(guard);
|
||||
writer
|
||||
});
|
||||
let file_writer = if config.file.enabled {
|
||||
let (file_writer, recovery_file_writer) = if config.file.enabled {
|
||||
let maximum_bytes = config
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
let file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (writer, guard) = NonBlockingBuilder::default()
|
||||
let recovery_file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (file_writer, file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-file")
|
||||
.finish(writer);
|
||||
guards.push(guard);
|
||||
Some(writer)
|
||||
.finish(file_writer);
|
||||
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-oauth-recovery-file")
|
||||
.finish(recovery_file_writer);
|
||||
guards.push(file_guard);
|
||||
guards.push(recovery_file_guard);
|
||||
(Some(file_writer), Some(recovery_file_writer))
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
@@ -89,14 +105,43 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
.with_span_list(true)
|
||||
.with_filter(file_filter)
|
||||
});
|
||||
let recovery_file_layer = recovery_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard { _writers: guards })
|
||||
}
|
||||
|
||||
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
directive.to_owned()
|
||||
@@ -107,6 +152,7 @@ fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> Str
|
||||
|
||||
struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -119,17 +165,20 @@ struct RotatingLogWriter {
|
||||
impl RotatingLogWriter {
|
||||
fn open(
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = Utc::now().date_naive();
|
||||
cleanup_log_files(&directory, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) = writable_log_segment(&directory, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, date, segment)?;
|
||||
cleanup_log_files(&directory, file_prefix, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) =
|
||||
writable_log_segment(&directory, file_prefix, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, file_prefix, date, segment)?;
|
||||
Ok(Self {
|
||||
directory,
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
maximum_files,
|
||||
@@ -157,10 +206,11 @@ impl RotatingLogWriter {
|
||||
} else {
|
||||
self.segment = self.segment.saturating_add(1);
|
||||
}
|
||||
self.file = open_log_segment(&self.directory, self.date, self.segment)?;
|
||||
self.file = open_log_segment(&self.directory, self.file_prefix, self.date, self.segment)?;
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
cleanup_log_files(
|
||||
&self.directory,
|
||||
self.file_prefix,
|
||||
self.date,
|
||||
self.retention_days,
|
||||
self.maximum_files,
|
||||
@@ -192,10 +242,11 @@ struct ManagedLogFile {
|
||||
|
||||
fn writable_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
maximum_bytes: u64,
|
||||
) -> io::Result<(usize, u64)> {
|
||||
let latest = managed_log_files(directory)?
|
||||
let latest = managed_log_files(directory, file_prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| entry.segment);
|
||||
@@ -210,26 +261,31 @@ fn writable_log_segment(
|
||||
}
|
||||
}
|
||||
|
||||
fn open_log_segment(directory: &Path, date: chrono::NaiveDate, segment: usize) -> io::Result<File> {
|
||||
fn open_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(directory.join(log_file_name(date, segment)))
|
||||
.open(directory.join(log_file_name(file_prefix, date, segment)))
|
||||
}
|
||||
|
||||
fn log_file_name(date: chrono::NaiveDate, segment: usize) -> String {
|
||||
fn log_file_name(file_prefix: &str, date: chrono::NaiveDate, segment: usize) -> String {
|
||||
if segment == 0 {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.log")
|
||||
format!("{file_prefix}.{date}.log")
|
||||
} else {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.{segment}.log")
|
||||
format!("{file_prefix}.{date}.{segment}.log")
|
||||
}
|
||||
}
|
||||
|
||||
fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
fn managed_log_files(directory: &Path, file_prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
|
||||
Ok(fs::read_dir(directory)?
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref()).map(
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref(), file_prefix).map(
|
||||
|(date, segment)| ManagedLogFile {
|
||||
date,
|
||||
segment,
|
||||
@@ -240,9 +296,9 @@ fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
fn parse_log_file_name(name: &str, file_prefix: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
let body = name
|
||||
.strip_prefix(&format!("{LOG_FILE_PREFIX}."))?
|
||||
.strip_prefix(&format!("{file_prefix}."))?
|
||||
.strip_suffix(".log")?;
|
||||
let (date, segment) = body.split_once('.').map_or((body, 0), |(date, segment)| {
|
||||
(date, segment.parse().ok().unwrap_or(usize::MAX))
|
||||
@@ -257,6 +313,7 @@ fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
today: chrono::NaiveDate,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -266,7 +323,7 @@ fn cleanup_log_files(
|
||||
.checked_sub_signed(chrono::Duration::days(retention_offset))
|
||||
.unwrap_or(chrono::NaiveDate::MIN);
|
||||
let mut retained = Vec::new();
|
||||
for entry in managed_log_files(directory)? {
|
||||
for entry in managed_log_files(directory, file_prefix)? {
|
||||
if entry.date < cutoff {
|
||||
fs::remove_file(entry.path)?;
|
||||
} else {
|
||||
|
||||
@@ -1,8 +1,21 @@
|
||||
use std::path::PathBuf;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use gateway_host::config::{FileLoggingConfig, HostConfig, ListenConfig, LoggingConfig};
|
||||
use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
|
||||
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
|
||||
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_sink() {
|
||||
let mut config = HostConfig {
|
||||
@@ -32,3 +45,115 @@ fn logging_requires_at_least_one_sink() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
write_oauth_recovery_log(PathBuf::from(
|
||||
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
let directory = tempfile::tempdir().expect("create log directory");
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env(
|
||||
"RUST_LOG",
|
||||
"off,logging_test_application=info,oauth_recovery=off",
|
||||
)
|
||||
.output()
|
||||
.expect("run logging child process");
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
|
||||
let application_log = read_log_file_set(directory.path(), APPLICATION_LOG_FILE_PREFIX);
|
||||
assert!(application_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(application_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
|
||||
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
|
||||
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(recovery_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
|
||||
}
|
||||
|
||||
fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
let config = HostConfig {
|
||||
listen: ListenConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
},
|
||||
logging: LoggingConfig {
|
||||
level: "off".to_owned(),
|
||||
stdout: false,
|
||||
file: FileLoggingConfig {
|
||||
enabled: true,
|
||||
directory,
|
||||
retention_days: 1,
|
||||
max_file_size_mb: 1,
|
||||
max_files: 1,
|
||||
},
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create logging runtime");
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
|
||||
tracing::info!(
|
||||
target: APPLICATION_LOG_TARGET,
|
||||
marker = APPLICATION_LOG_MARKER,
|
||||
"application file test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
let mut files = fs::read_dir(directory)
|
||||
.expect("read log directory")
|
||||
.map(|entry| entry.expect("read log entry").path())
|
||||
.filter(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
files.sort();
|
||||
|
||||
assert!(!files.is_empty(), "expected log file set {file_prefix}");
|
||||
files
|
||||
.into_iter()
|
||||
.map(|path| fs::read_to_string(path).expect("read log file"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn json_target_field(target: &str) -> String {
|
||||
format!(r#""target":"{target}""#)
|
||||
}
|
||||
|
||||
@@ -315,6 +315,26 @@ pub fn is_rate_limit_header_name(name: &str) -> bool {
|
||||
|| rate_limit_id_from_header_name(&normalized).is_some()
|
||||
}
|
||||
|
||||
/// 判断响应头是否为仅供代理本地观测的 Codex 额度信号。
|
||||
///
|
||||
/// 这类头会让下游官方 Codex 客户端显示账户额度提示,因此不能穿过账号隔离边界。
|
||||
/// 通用限流头(例如 `retry-after`)和非额度的 Codex 头仍允许下发。
|
||||
pub fn is_codex_quota_header_name(name: &str) -> bool {
|
||||
let normalized = name.trim().to_ascii_lowercase();
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"x-codex-credits-has-credits"
|
||||
| "x-codex-credits-unlimited"
|
||||
| "x-codex-credits-balance"
|
||||
| "x-codex-active-limit"
|
||||
| "x-codex-plan-type"
|
||||
| "x-codex-promo-message"
|
||||
| "x-codex-rate-limit-reached-type"
|
||||
| "x-codex-primary-over-secondary-limit-percent"
|
||||
) || rate_limit_id_from_header_name(&normalized)
|
||||
.is_some_and(|limit_id| limit_id == "codex" || limit_id.starts_with("codex_"))
|
||||
}
|
||||
|
||||
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
|
||||
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
if value.get("type").and_then(Value::as_str) != Some("codex.rate_limits") {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use gateway_protocol::openai::events::{
|
||||
RateLimitWindow, TokenUsage, extract_sse_usage, extract_usage, is_rate_limit_header_name,
|
||||
parse_rate_limit_headers, parse_rate_limits_event, rate_limits_to_header_pairs,
|
||||
retry_after_seconds_from_body,
|
||||
RateLimitWindow, TokenUsage, extract_sse_usage, extract_usage, is_codex_quota_header_name,
|
||||
is_rate_limit_header_name, parse_rate_limit_headers, parse_rate_limits_event,
|
||||
rate_limits_to_header_pairs, retry_after_seconds_from_body,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -537,3 +537,21 @@ fn rate_limit_header_filter_should_accept_only_supported_domain_headers() {
|
||||
[true, true, true, false, false]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_quota_header_filter_should_only_match_client_quota_signals() {
|
||||
assert_eq!(
|
||||
[
|
||||
"X-CoDeX-Primary-Used-Percent",
|
||||
"x-codex-code-review-primary-reset-at",
|
||||
"x-codex-credits-balance",
|
||||
"x-codex-primary-over-secondary-limit-percent",
|
||||
"retry-after",
|
||||
"x-ratelimit-remaining",
|
||||
"x-codex-safety-buffering-enabled",
|
||||
"x-codex-turn-state",
|
||||
]
|
||||
.map(is_codex_quota_header_name),
|
||||
[true, true, true, true, false, false, false, false]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.23"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "process", "time"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "net", "process", "time"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -385,7 +385,13 @@ async fn upload_parts(
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "上传并发闸门关闭".to_owned())
|
||||
})?;
|
||||
let offset = (part - 1) * (PART_SIZE as u64);
|
||||
let bytes = read_part(&source, offset, PART_SIZE).await?;
|
||||
let remaining = file_size.checked_sub(offset).ok_or_else(|| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片偏移超出归档大小".to_owned())
|
||||
})?;
|
||||
let part_len = usize::try_from(remaining.min(PART_SIZE as u64)).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片大小溢出".to_owned())
|
||||
})?;
|
||||
let bytes = read_part(&source, offset, part_len).await?;
|
||||
let part_number = i32::try_from(part).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片号溢出".to_owned())
|
||||
})?;
|
||||
@@ -450,20 +456,19 @@ async fn upload_parts(
|
||||
.map_err(map_s3_error)
|
||||
}
|
||||
|
||||
/// 从文件中读取一个分片(最多 `PART_SIZE` 字节)。
|
||||
async fn read_part(source: &Path, offset: u64, max_len: usize) -> Result<Vec<u8>, BackupError> {
|
||||
/// 从文件中读取指定长度的完整分片。
|
||||
async fn read_part(source: &Path, offset: u64, len: usize) -> Result<Vec<u8>, BackupError> {
|
||||
let mut file = tokio::fs::File::open(source)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "打开暂存归档失败".to_owned()))?;
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "定位暂存归档失败".to_owned()))?;
|
||||
let mut buffer = vec![0_u8; max_len];
|
||||
let n = file
|
||||
.read(&mut buffer)
|
||||
let mut buffer = vec![0_u8; len];
|
||||
// Tokio 默认单次文件读取最多 2 MiB;分片必须读取完整,不能把短读当作末尾分片上传。
|
||||
file.read_exact(&mut buffer)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?;
|
||||
buffer.truncate(n);
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountStore;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 已完成连接、迁移与 cooldown hydration 的 Store 能力集合。
|
||||
/// 已完成连接、迁移与 hydration 的 Store 能力集合。
|
||||
pub struct StoreBundle {
|
||||
admin_ports: AdminStorePorts,
|
||||
core_ports: CoreStorePorts,
|
||||
@@ -61,12 +64,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let account_store = Arc::new(redis::CooldownCachingProviderAccountStore::new(
|
||||
provider_accounts,
|
||||
cooldowns.clone(),
|
||||
));
|
||||
let hydrated = account_store.hydrate(std::time::SystemTime::now()).await;
|
||||
tracing::info!(hydrated, "账号 cooldown Redis 热缓存重建完成");
|
||||
let account_store: Arc<dyn ProviderAccountStore> = provider_accounts;
|
||||
|
||||
let credential_leases =
|
||||
redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?;
|
||||
@@ -88,7 +86,10 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
)?);
|
||||
|
||||
let admin_ports = AdminStorePorts::new(
|
||||
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
|
||||
Arc::new(postgres::PgAdminAccountStore::new(
|
||||
pool.clone(),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminAuthStoreAdapter {
|
||||
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
|
||||
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
|
||||
@@ -101,6 +102,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
Arc::new(postgres::PgAdminObservabilityStore::new(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminSettingsStoreAdapter {
|
||||
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -7,12 +12,82 @@ use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepositor
|
||||
#[derive(Clone)]
|
||||
pub struct PgObservabilityRepository {
|
||||
pool: PgPool,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgObservabilityRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,9 +103,13 @@ pub struct PgAdminObservabilityStore {
|
||||
|
||||
impl PgAdminObservabilityStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository: PgObservabilityRepository::new(pool),
|
||||
repository: PgObservabilityRepository::new(pool, cooldowns),
|
||||
runtime_signals,
|
||||
}
|
||||
}
|
||||
@@ -60,13 +139,15 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
page: ObservabilityPageNumber::new(1)?,
|
||||
page_size: ObservabilityPageSize::new(10)?,
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, provider_accounts) = futures::try_join!(
|
||||
// 每批最多三路并发,既缩短概览渲染路径,也不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, totals) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
provider_account_metrics(&self.pool, range.end),
|
||||
dashboard_totals(&self.pool),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
@@ -76,6 +157,7 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
totals,
|
||||
provider_accounts,
|
||||
trend,
|
||||
account_usage,
|
||||
|
||||
@@ -103,6 +103,7 @@ pub(crate) fn admin_dashboard_observation(
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
totals,
|
||||
provider_accounts,
|
||||
trend,
|
||||
account_usage,
|
||||
@@ -112,6 +113,7 @@ pub(crate) fn admin_dashboard_observation(
|
||||
range: admin_range(range),
|
||||
requests: admin_request_metrics(requests)?,
|
||||
attempts: admin_attempt_metrics(attempts)?,
|
||||
totals: admin_dashboard_totals(totals)?,
|
||||
provider_accounts: admin_account_pool_metrics(provider_accounts),
|
||||
trend: trend
|
||||
.into_iter()
|
||||
@@ -128,6 +130,18 @@ pub(crate) fn admin_dashboard_observation(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_dashboard_totals(
|
||||
totals: DashboardTotals,
|
||||
) -> AdminStoreResult<admin_observability::DashboardTotals> {
|
||||
Ok(admin_observability::DashboardTotals {
|
||||
request_count: totals.request_count,
|
||||
input_tokens: totals.input_tokens,
|
||||
cached_tokens: totals.cached_tokens,
|
||||
total_tokens: totals.total_tokens,
|
||||
billing_usd: admin_optional_decimal_amount(totals.billing_usd)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_range(range: ObservabilityRange) -> admin_observability::TimeRange {
|
||||
admin_observability::TimeRange {
|
||||
start: range.start,
|
||||
@@ -263,7 +277,9 @@ pub(crate) const fn admin_account_pool_metrics(
|
||||
enabled: metrics.enabled,
|
||||
unavailable: metrics.unavailable,
|
||||
active: metrics.active,
|
||||
rate_limited: metrics.rate_limited,
|
||||
expired: metrics.expired,
|
||||
invalid: metrics.invalid,
|
||||
quota_exhausted: metrics.quota_exhausted,
|
||||
refreshing: None,
|
||||
disabled: metrics.disabled,
|
||||
|
||||
@@ -366,7 +366,9 @@ pub struct ProviderAccountMetrics {
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
@@ -475,11 +477,21 @@ impl ProviderAccountUsageQuery {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardObservation {
|
||||
pub range: ObservabilityRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: ProviderAccountMetrics,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub account_usage: Vec<ProviderAccountUsageObservation>,
|
||||
|
||||
@@ -65,6 +65,33 @@ pub(crate) async fn request_metrics(
|
||||
request_metrics_from_row(&row)
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTotals> {
|
||||
// 卡片脚注的“总计”覆盖全部历史;只聚合页面实际展示的字段,避免重复计算
|
||||
// 当前区间指标所需的延迟分位数。
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select count(*)::bigint as request_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
sum(cost_amount) filter (where {fact} and cost_currency = 'USD')::text
|
||||
as billing_usd
|
||||
from model_requests mr"
|
||||
));
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load dashboard totals"))?;
|
||||
Ok(DashboardTotals {
|
||||
request_count: unsigned(&row, "request_count")?,
|
||||
input_tokens: unsigned(&row, "input_tokens")?,
|
||||
cached_tokens: unsigned(&row, "cached_tokens")?,
|
||||
total_tokens: unsigned(&row, "total_tokens")?,
|
||||
billing_usd: optional_decimal(&row, "billing_usd")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn request_metric_series(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
@@ -348,29 +375,28 @@ pub(crate) async fn provider_account_metrics(
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"with normalized as (
|
||||
select enabled,
|
||||
select enabled, availability, quota_limit_reached,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability = 'invalid' then 'invalid'
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where status <> 'active')::bigint as unavailable,
|
||||
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'invalid')::bigint as invalid,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
@@ -385,7 +411,9 @@ pub(crate) async fn provider_account_metrics(
|
||||
enabled: unsigned(&row, "enabled")?,
|
||||
unavailable: unsigned(&row, "unavailable")?,
|
||||
active: unsigned(&row, "active")?,
|
||||
rate_limited: unsigned(&row, "rate_limited")?,
|
||||
expired: unsigned(&row, "expired")?,
|
||||
invalid: unsigned(&row, "invalid")?,
|
||||
quota_exhausted: unsigned(&row, "quota_exhausted")?,
|
||||
disabled: unsigned(&row, "disabled")?,
|
||||
banned: unsigned(&row, "banned")?,
|
||||
@@ -402,13 +430,7 @@ pub(crate) async fn active_provider_account_ids(
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
@@ -430,6 +452,35 @@ pub(crate) async fn active_provider_account_ids(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
//! `gateway-admin` 账号端口的 PostgreSQL adapter。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。
|
||||
@@ -11,19 +17,49 @@ pub struct PgAdminAccountStore {
|
||||
accounts: PgProviderAccountRepository,
|
||||
observability: PgObservabilityRepository,
|
||||
control_plane: PgControlPlaneRepository,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgAdminAccountStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
pub fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self {
|
||||
pool: pool.clone(),
|
||||
accounts: PgProviderAccountRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone(), cooldowns.clone()),
|
||||
control_plane: PgControlPlaneRepository::new(pool),
|
||||
cooldowns,
|
||||
}
|
||||
}
|
||||
|
||||
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
|
||||
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
|
||||
async fn load_rate_limited_until(
|
||||
&self,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let mut map = BTreeMap::new();
|
||||
for account in accounts {
|
||||
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
|
||||
continue;
|
||||
};
|
||||
if cooldown.credential_revision().get() != account.credential_revision.get() {
|
||||
continue;
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until > SystemTime::now() {
|
||||
map.insert(account.id.clone(), until);
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
async fn usage_observations(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
@@ -242,11 +278,17 @@ impl AccountStore for PgAdminAccountStore {
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let now = Utc::now();
|
||||
let summary = admin_account_summary(&accounts, now);
|
||||
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
|
||||
let rate_limited_until = self
|
||||
.load_rate_limited_until(&accounts)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
|
||||
let mut items = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let status = admin_account_status(&account, now);
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
let status = admin_account_status(&account, now, until);
|
||||
account_matches_admin_query(&account, status, &query).then_some(
|
||||
AdminAccountListItem {
|
||||
account,
|
||||
|
||||
@@ -19,7 +19,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
provider_kind: account.account.provider().as_str().to_owned(),
|
||||
name: account.account.name().to_owned(),
|
||||
email: account.account.email().map(str::to_owned),
|
||||
upstream_user_id: account.account.upstream_user_id().to_owned(),
|
||||
upstream_user_id: account.account.upstream_user_id().map(str::to_owned),
|
||||
upstream_account_id: account.account.upstream_account_id().map(str::to_owned),
|
||||
plan_type: account.account.plan_type().map(str::to_owned),
|
||||
authentication_kind: account.account.authentication_kind().to_owned(),
|
||||
@@ -32,8 +32,6 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
next_refresh_at: account.account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.account.enabled(),
|
||||
availability: account.account.availability(),
|
||||
availability_reason: None,
|
||||
cooldown_until: account.account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -165,7 +163,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.map(|account| account.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let rows = sqlx::query(
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at \
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at, \
|
||||
quota_limit_reached \
|
||||
from provider_accounts \
|
||||
where id = any($1) and provider_quota_json is not null",
|
||||
)
|
||||
@@ -193,12 +192,16 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
let observed_at = row
|
||||
.try_get::<DateTime<Utc>, _>("quota_observed_at")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let limit_reached = row
|
||||
.try_get::<bool, _>("quota_limit_reached")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
Ok(QuotaObservation {
|
||||
account_id: CoreProviderAccountId::new(account_id)
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
expected_revision: revision,
|
||||
quota: Some(quota),
|
||||
observed_at: Some(observed_at.into()),
|
||||
limit_reached: Some(limit_reached),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -225,6 +228,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
Revision::new(observation.expected_revision.get()).map_err(core_store_error)?,
|
||||
quota,
|
||||
observation.observed_at.map(DateTime::<Utc>::from),
|
||||
observation.limit_reached,
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
@@ -242,9 +246,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
expected_revision: Revision::new(change.expected_revision.get())
|
||||
.map_err(core_store_error)?,
|
||||
availability: change.availability,
|
||||
availability_reason: change.reason,
|
||||
cooldown_until: change.cooldown_until.map(DateTime::<Utc>::from),
|
||||
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
|
||||
message: change.message,
|
||||
})
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
use gateway_admin::model::accounts::AccountStatusSignals;
|
||||
|
||||
pub(crate) struct AdminAccountListItem {
|
||||
pub(crate) account: ProviderAccountSummary,
|
||||
pub(crate) status: AdminAccountStatus,
|
||||
@@ -45,7 +47,7 @@ pub(crate) fn account_matches_admin_query(
|
||||
Some(account.name.as_str()),
|
||||
account.email.as_deref(),
|
||||
account.upstream_account_id.as_deref(),
|
||||
Some(account.upstream_user_id.as_str()),
|
||||
account.upstream_user_id.as_deref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
@@ -58,50 +60,48 @@ pub(crate) fn account_matches_admin_query(
|
||||
pub(crate) fn admin_account_status(
|
||||
account: &ProviderAccountSummary,
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AdminAccountStatus {
|
||||
if !account.enabled {
|
||||
AdminAccountStatus::Disabled
|
||||
} else if account.availability == AccountAvailability::Banned {
|
||||
AdminAccountStatus::Banned
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == AccountAvailability::Expired
|
||||
|| account.availability == AccountAvailability::Invalid
|
||||
|| account
|
||||
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AdminAccountStatus::Expired
|
||||
} else {
|
||||
AdminAccountStatus::Active
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: account.quota_limit_reached,
|
||||
rate_limited_until,
|
||||
}
|
||||
.derive()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_summary(
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: &BTreeMap<String, SystemTime>,
|
||||
) -> AccountSummary {
|
||||
let mut summary = AccountSummary {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
active: 0,
|
||||
normal: 0,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
};
|
||||
for account in accounts {
|
||||
match admin_account_status(account, now) {
|
||||
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
match admin_account_status(account, now, until) {
|
||||
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
|
||||
AdminAccountStatus::QuotaExhausted => {
|
||||
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Expired
|
||||
| AdminAccountStatus::Disabled
|
||||
| AdminAccountStatus::Banned => {
|
||||
summary.unavailable = summary.unavailable.saturating_add(1);
|
||||
AdminAccountStatus::RateLimited => {
|
||||
summary.rate_limited = summary.rate_limited.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Disabled => {
|
||||
summary.disabled = summary.disabled.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Error => summary.error = summary.error.saturating_add(1),
|
||||
}
|
||||
}
|
||||
summary
|
||||
@@ -118,9 +118,7 @@ pub(crate) fn sort_admin_account_items(
|
||||
items.sort_by(|left, right| {
|
||||
let ordering = match sort.field {
|
||||
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
|
||||
AdminAccountSortField::Status => {
|
||||
admin_account_status_name(left.status).cmp(admin_account_status_name(right.status))
|
||||
}
|
||||
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
|
||||
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
|
||||
AdminAccountSortField::Usage => left
|
||||
.usage
|
||||
@@ -150,16 +148,6 @@ pub(crate) fn sort_admin_account_items(
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
|
||||
match status {
|
||||
AdminAccountStatus::Active => "active",
|
||||
AdminAccountStatus::Expired => "expired",
|
||||
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
AdminAccountStatus::Disabled => "disabled",
|
||||
AdminAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_record(
|
||||
summary: ProviderAccountSummary,
|
||||
) -> AdminStoreResult<AccountRecord> {
|
||||
@@ -184,9 +172,8 @@ pub(crate) fn admin_account_record(
|
||||
next_refresh_at: summary.next_refresh_at,
|
||||
enabled: summary.enabled,
|
||||
availability: summary.availability,
|
||||
availability_reason: summary.availability_reason,
|
||||
cooldown_until: summary.cooldown_until,
|
||||
availability_observed_at: summary.availability_observed_at,
|
||||
last_error_message: summary.last_error_message,
|
||||
quota_observed_at: summary.quota_observed_at,
|
||||
created_at: summary.created_at,
|
||||
updated_at: summary.updated_at,
|
||||
@@ -211,8 +198,6 @@ pub(crate) fn prepared_account(
|
||||
next_refresh_at: credential.next_refresh_at,
|
||||
enabled: credential.enabled,
|
||||
availability: credential.availability,
|
||||
availability_reason: credential.availability_reason,
|
||||
cooldown_until: credential.cooldown_until,
|
||||
availability_observed_at: credential.availability_observed_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
str::FromStr,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -27,6 +27,7 @@ pub trait ProviderAccountRepository: Send + Sync {
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
) -> StoreResult<bool>;
|
||||
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool>;
|
||||
}
|
||||
@@ -93,8 +94,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
"select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, availability_observed_at,
|
||||
quota_observed_at, created_at, updated_at
|
||||
availability_observed_at,
|
||||
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
|
||||
from provider_accounts
|
||||
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
|
||||
order by provider_kind, name, id",
|
||||
@@ -109,16 +110,21 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> {
|
||||
account.validate()?;
|
||||
let availability = if account.upstream_user_id.is_some() {
|
||||
account.availability
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
};
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, availability_reason, cooldown_until, provider_quota_json,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, null, $15, null, $16, null, now(), greatest(now(), $16)
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
)",
|
||||
)
|
||||
.bind(account.id)
|
||||
@@ -134,8 +140,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(account.access_token_expires_at)
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(account.availability.as_str())
|
||||
.bind(account.cooldown_until)
|
||||
.bind(availability.as_str())
|
||||
.bind(account.availability_observed_at)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
@@ -209,23 +214,28 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
update.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = case when enabled then $3 else availability end,
|
||||
availability_reason = case when enabled then $4 else availability_reason end,
|
||||
cooldown_until = case when enabled then $5 else cooldown_until end,
|
||||
set availability = case
|
||||
when enabled and upstream_user_id is not null then $3
|
||||
else availability
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when enabled then $6
|
||||
when enabled and upstream_user_id is not null then $4
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $6) else updated_at end
|
||||
last_error_message = case
|
||||
when enabled and upstream_user_id is not null then
|
||||
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
|
||||
else last_error_message
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $4) else updated_at end
|
||||
where id = $1 and credential_revision = $2
|
||||
and (availability_observed_at is null or availability_observed_at <= $6)",
|
||||
and (availability_observed_at is null or availability_observed_at <= $4)",
|
||||
)
|
||||
.bind(update.account_id)
|
||||
.bind(to_i64(update.expected_revision.get())?)
|
||||
.bind(update.availability.as_str())
|
||||
.bind(update.availability_reason)
|
||||
.bind(update.cooldown_until)
|
||||
.bind(update.availability_observed_at)
|
||||
.bind(update.message.as_deref())
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("apply provider account state"))?;
|
||||
@@ -251,6 +261,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "account_id", account_id)?;
|
||||
if quota.is_some() != observed_at.is_some() {
|
||||
@@ -262,6 +273,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set provider_quota_json = $3, quota_observed_at = $4,
|
||||
quota_limit_reached = coalesce($5, quota_limit_reached),
|
||||
updated_at = greatest(now(), coalesce($4, now()))
|
||||
where id = $1 and credential_revision = $2
|
||||
and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)",
|
||||
@@ -270,6 +282,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(to_i64(expected_revision.get())?)
|
||||
.bind(quota.map(|value| value.as_value()))
|
||||
.bind(observed_at)
|
||||
.bind(limit_reached)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
|
||||
@@ -461,16 +474,21 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
account: &NewProviderAccount,
|
||||
) -> StoreResult<String> {
|
||||
account.validate()?;
|
||||
let availability = if account.upstream_user_id.is_some() {
|
||||
account.availability
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
};
|
||||
let imported_id = sqlx::query_scalar::<_, String>(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, availability_reason, cooldown_until, provider_quota_json,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, $15, $16, null, $17, null, now(), greatest(now(), $17)
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
)
|
||||
on conflict (
|
||||
provider_kind,
|
||||
@@ -488,8 +506,6 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
next_refresh_at = excluded.next_refresh_at,
|
||||
enabled = excluded.enabled,
|
||||
availability = excluded.availability,
|
||||
availability_reason = excluded.availability_reason,
|
||||
cooldown_until = excluded.cooldown_until,
|
||||
provider_quota_json = null,
|
||||
availability_observed_at = excluded.availability_observed_at,
|
||||
quota_observed_at = null,
|
||||
@@ -509,9 +525,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
.bind(account.access_token_expires_at)
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(account.availability.as_str())
|
||||
.bind(&account.availability_reason)
|
||||
.bind(account.cooldown_until)
|
||||
.bind(availability.as_str())
|
||||
.bind(account.availability_observed_at)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
@@ -562,18 +576,9 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
|
||||
availability = case
|
||||
when not enabled then availability
|
||||
when availability <> 'quota_exhausted' then 'ready'
|
||||
else availability
|
||||
end,
|
||||
availability_reason = case
|
||||
when not enabled then availability_reason
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else availability_reason
|
||||
end,
|
||||
cooldown_until = case
|
||||
when not enabled then cooldown_until
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else cooldown_until
|
||||
when availability = 'quota_exhausted' then availability
|
||||
when coalesce($12::text, upstream_user_id) is not null then 'ready'
|
||||
else 'unknown'
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when not enabled then availability_observed_at
|
||||
|
||||
@@ -70,33 +70,13 @@ pub(crate) fn parse_availability(value: &str) -> StoreResult<AccountAvailability
|
||||
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_availability_cooldown(
|
||||
availability: AccountAvailability,
|
||||
has_cooldown_until: bool,
|
||||
) -> StoreResult<()> {
|
||||
if availability == AccountAvailability::Cooldown && !has_cooldown_until {
|
||||
return Err(invalid("cooldown availability requires cooldown_until"));
|
||||
}
|
||||
if has_cooldown_until
|
||||
&& !matches!(
|
||||
availability,
|
||||
AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted
|
||||
)
|
||||
{
|
||||
return Err(invalid(
|
||||
"cooldown_until is only allowed for cooldown or quota_exhausted availability",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountSummary {
|
||||
pub id: String,
|
||||
pub provider_kind: String,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -106,10 +86,12 @@ pub struct ProviderAccountSummary {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -141,7 +123,7 @@ pub struct NewProviderAccount {
|
||||
pub provider_kind: String,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -151,8 +133,6 @@ pub struct NewProviderAccount {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -173,7 +153,6 @@ impl fmt::Debug for NewProviderAccount {
|
||||
.field("name", &self.name)
|
||||
.field("enabled", &self.enabled)
|
||||
.field("availability", &self.availability)
|
||||
.field("cooldown_until", &self.cooldown_until)
|
||||
.field("provider_credentials_json", &"[REDACTED]")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
@@ -184,12 +163,13 @@ impl NewProviderAccount {
|
||||
require_nonempty(ENTITY, "id", &self.id)?;
|
||||
require_nonempty(ENTITY, "provider_kind", &self.provider_kind)?;
|
||||
require_nonempty(ENTITY, "name", &self.name)?;
|
||||
require_nonempty(ENTITY, "upstream_user_id", &self.upstream_user_id)?;
|
||||
if let Some(upstream_user_id) = &self.upstream_user_id {
|
||||
require_nonempty(ENTITY, "upstream_user_id", upstream_user_id)?;
|
||||
}
|
||||
require_nonempty(ENTITY, "authentication_kind", &self.authentication_kind)?;
|
||||
if !self.has_refresh_token && self.next_refresh_at.is_some() {
|
||||
return Err(invalid("next_refresh_at requires a refresh token"));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
validate_object_size(
|
||||
"provider_credentials_json",
|
||||
&self.provider_credentials_json,
|
||||
@@ -321,11 +301,11 @@ impl fmt::Debug for ProviderCredentialUpdate {
|
||||
pub struct ProviderAccountObservation {
|
||||
pub account_id: String,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub provider_quota_json: Option<JsonObject>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -333,15 +313,14 @@ pub struct ProviderAccountStateUpdate {
|
||||
pub account_id: String,
|
||||
pub expected_revision: Revision,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl ProviderAccountStateUpdate {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -349,7 +328,6 @@ impl ProviderAccountStateUpdate {
|
||||
impl ProviderAccountObservation {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and quota_observed_at must agree"));
|
||||
}
|
||||
@@ -363,14 +341,14 @@ impl ProviderAccountObservation {
|
||||
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, provider_quota_json,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts where id = $1";
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, provider_quota_json,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts
|
||||
where id = any($1::text[]) and provider_kind = $2
|
||||
@@ -420,11 +398,7 @@ pub(crate) fn core_account_from_summary(
|
||||
summary.upstream_account_id,
|
||||
summary.plan_type,
|
||||
)
|
||||
.with_runtime_state(
|
||||
summary.enabled,
|
||||
summary.availability,
|
||||
summary.cooldown_until.map(Into::into),
|
||||
)
|
||||
.with_runtime_state(summary.enabled, summary.availability)
|
||||
.with_refresh_schedule(
|
||||
summary.has_refresh_token,
|
||||
summary.next_refresh_at.map(Into::into),
|
||||
@@ -474,10 +448,10 @@ pub(crate) fn account_summary_from_row(
|
||||
next_refresh_at: get(&row, "next_refresh_at")?,
|
||||
enabled: get(&row, "enabled")?,
|
||||
availability: parse_availability(&availability)?,
|
||||
availability_reason: get(&row, "availability_reason")?,
|
||||
cooldown_until: get(&row, "cooldown_until")?,
|
||||
availability_observed_at: get(&row, "availability_observed_at")?,
|
||||
quota_observed_at: get(&row, "quota_observed_at")?,
|
||||
quota_limit_reached: get(&row, "quota_limit_reached")?,
|
||||
last_error_message: get(&row, "last_error_message")?,
|
||||
created_at: get(&row, "created_at")?,
|
||||
updated_at: get(&row, "updated_at")?,
|
||||
})
|
||||
|
||||
@@ -225,19 +225,6 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
update.validate()?;
|
||||
let refresh_margin_seconds =
|
||||
i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?;
|
||||
let previous_refresh_margin_seconds = sqlx::query_scalar::<_, i64>(
|
||||
"select refresh_margin_seconds
|
||||
from runtime_settings
|
||||
where id = 1
|
||||
for update",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("lock runtime settings for update"))?
|
||||
.ok_or_else(|| StoreError::NotFound {
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update runtime_settings
|
||||
set config_revision = config_revision + 1,
|
||||
@@ -272,35 +259,9 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
if previous_refresh_margin_seconds != refresh_margin_seconds {
|
||||
reschedule_provider_refreshes_in_transaction(transaction, refresh_margin_seconds).await?;
|
||||
}
|
||||
Revision::new(u64::try_from(next).map_err(|_| invalid_numeric())?)
|
||||
}
|
||||
|
||||
async fn reschedule_provider_refreshes_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
refresh_margin_seconds: i64,
|
||||
) -> StoreResult<()> {
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set next_refresh_at = case
|
||||
when access_token_expires_at <= now()
|
||||
or extract(epoch from (access_token_expires_at - now())) <= $1
|
||||
then now()
|
||||
else access_token_expires_at - ($1::double precision * interval '1 second')
|
||||
end,
|
||||
updated_at = now()
|
||||
where has_refresh_token
|
||||
and access_token_expires_at is not null",
|
||||
)
|
||||
.bind(refresh_margin_seconds)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("reschedule provider credential refreshes"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn bump_config_revision_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> StoreResult<Revision> {
|
||||
|
||||
@@ -1,22 +1,13 @@
|
||||
//! 请求调度用的可丢失账号 cooldown Redis 存储。
|
||||
|
||||
use std::{sync::Arc, time::SystemTime};
|
||||
//! 请求调度用的可丢失 Provider 级 cooldown Redis 存储。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
},
|
||||
error::StoreError as CoreStoreError,
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError, ProviderStoreErrorKind,
|
||||
},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
|
||||
@@ -85,6 +76,8 @@ pub trait CredentialCooldownRepository: Send + Sync {
|
||||
provider_account_id: &str,
|
||||
through_revision: Revision,
|
||||
) -> StoreResult<bool>;
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
async fn delete_account_cooldowns(&self, provider_account_id: &str) -> StoreResult<bool>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -227,6 +220,50 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
self.invalidate_at_key(self.key(provider_account_id)?, through_revision)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_account_cooldowns(&self, provider_account_id: &str) -> StoreResult<bool> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
provider_account_id,
|
||||
)?;
|
||||
// 账号删除:清除该账号的 account key 与全部 model-scoped key。
|
||||
// 用 SCAN 精确匹配命名空间内该账号前缀,避免 KEYS 阻塞。
|
||||
let mut connection = self.connection.clone();
|
||||
let account_key = self.key(provider_account_id)?;
|
||||
let mut keys = vec![account_key.clone()];
|
||||
let pattern = format!(
|
||||
"{}:account:{}:cooldown:*",
|
||||
self.namespace,
|
||||
resource_fingerprint("credential cooldown", provider_account_id)?
|
||||
);
|
||||
let mut cursor = 0_i64;
|
||||
loop {
|
||||
let (next, found): (i64, Vec<String>) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(&pattern)
|
||||
.arg("COUNT")
|
||||
.arg(100)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("scan account cooldown keys"))?;
|
||||
keys.extend(found);
|
||||
cursor = next;
|
||||
if cursor == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let removed: i64 = redis::cmd("DEL")
|
||||
.arg(keys)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("delete account cooldown keys"))?;
|
||||
Ok(removed > 0)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
@@ -350,216 +387,17 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_unavailable("clear scoped credential cooldown"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号状态写入到可丢失 cooldown cache 的桥接 adapter。
|
||||
pub struct CooldownCachingProviderAccountStore {
|
||||
authoritative: Arc<dyn ProviderAccountStore>,
|
||||
cooldowns: Arc<dyn CredentialCooldownRepository>,
|
||||
}
|
||||
|
||||
impl CooldownCachingProviderAccountStore {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
authoritative: Arc<dyn ProviderAccountStore>,
|
||||
cooldowns: Arc<dyn CredentialCooldownRepository>,
|
||||
) -> Self {
|
||||
Self {
|
||||
authoritative,
|
||||
cooldowns,
|
||||
}
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
self.delete_account_cooldowns(account_id.as_str())
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("clear all credential cooldowns"))
|
||||
})
|
||||
}
|
||||
|
||||
/// 启动时从 PostgreSQL 尽力重建仍有效的 cooldown cache。
|
||||
pub async fn hydrate(&self, now: SystemTime) -> u64 {
|
||||
let Ok(accounts) = self.authoritative.list_accounts().await else {
|
||||
return 0;
|
||||
};
|
||||
let mut hydrated = 0_u64;
|
||||
for account in accounts {
|
||||
let Some(cooldown) = projection(
|
||||
account.id(),
|
||||
account.revision(),
|
||||
account.availability(),
|
||||
account.cooldown_until(),
|
||||
now,
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if matches!(
|
||||
self.cooldowns.cache_credential_cooldown(&cooldown).await,
|
||||
Ok(true)
|
||||
) {
|
||||
hydrated = hydrated.saturating_add(1);
|
||||
}
|
||||
}
|
||||
hydrated
|
||||
}
|
||||
|
||||
async fn mirror(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) {
|
||||
let Some(cooldown) = projection(
|
||||
account_id,
|
||||
credential_revision,
|
||||
availability,
|
||||
cooldown_until,
|
||||
SystemTime::now(),
|
||||
) else {
|
||||
let Ok(revision) = Revision::new(credential_revision.get()) else {
|
||||
return;
|
||||
};
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account_id.as_str(), revision)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let _ = self.cooldowns.cache_credential_cooldown(&cooldown).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for CooldownCachingProviderAccountStore {
|
||||
async fn create_account(&self, account: NewProviderAccount) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.create_account(account).await
|
||||
}
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.get_account(account).await
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.list_accounts().await
|
||||
}
|
||||
|
||||
async fn list_for_provider(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.list_for_provider(provider).await
|
||||
}
|
||||
|
||||
async fn load_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
self.authoritative
|
||||
.load_credential(account, expected_revision)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
self.authoritative.load_current_credential(account).await
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
) -> Result<CredentialCasOutcome, CoreStoreError> {
|
||||
let account_id = update.account_id().clone();
|
||||
let outcome = self
|
||||
.authoritative
|
||||
.compare_and_swap_credential(update)
|
||||
.await?;
|
||||
if let CredentialCasOutcome::Updated(credential_revision) = outcome
|
||||
&& let Ok(revision) = Revision::new(credential_revision.get())
|
||||
{
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account_id.as_str(), revision)
|
||||
.await;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn get_quotas(
|
||||
&self,
|
||||
accounts: &[ProviderAccountId],
|
||||
) -> Result<Vec<QuotaObservation>, CoreStoreError> {
|
||||
self.authoritative.get_quotas(accounts).await
|
||||
}
|
||||
|
||||
async fn compare_and_swap_quota(
|
||||
&self,
|
||||
observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
self.authoritative.compare_and_swap_quota(observation).await
|
||||
}
|
||||
|
||||
async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), CoreStoreError> {
|
||||
let account_id = change.account_id.clone();
|
||||
let credential_revision = change.expected_revision;
|
||||
let availability = change.availability;
|
||||
let cooldown_until = change.cooldown_until;
|
||||
self.authoritative.apply_state_change(change).await?;
|
||||
let Some(account) = self.authoritative.get_account(&account_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.mirror(
|
||||
&account_id,
|
||||
credential_revision,
|
||||
availability,
|
||||
cooldown_until,
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.update_account(update).await
|
||||
}
|
||||
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
enabled: bool,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.set_enabled(account, enabled).await
|
||||
}
|
||||
|
||||
async fn delete_account(&self, account: &ProviderAccountId) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.delete_account(account).await?;
|
||||
if let Ok(through_revision) = Revision::new(u64::MAX) {
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account.as_str(), through_revision)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn projection(
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
) -> Option<CredentialCooldown> {
|
||||
let cooldown_until = cooldown_until
|
||||
.filter(|_| availability == AccountAvailability::Cooldown)
|
||||
.filter(|until| *until > now)
|
||||
.map(DateTime::<Utc>::from)?;
|
||||
Some(CredentialCooldown {
|
||||
provider_account_id: account_id.as_str().to_owned(),
|
||||
credential_revision: Revision::new(credential_revision.get()).ok()?,
|
||||
cooldown_until,
|
||||
})
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> StoreError {
|
||||
|
||||
@@ -512,6 +512,7 @@ impl RedisProviderLeaseCoordinator {
|
||||
last_started_at: signal.last_started_at.map(Into::into),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod postgres;
|
||||
mod redis;
|
||||
mod s3;
|
||||
mod support;
|
||||
|
||||
@@ -156,7 +156,12 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (10, 1));
|
||||
assert_eq!(first_table_count, 10);
|
||||
assert_eq!(
|
||||
migration_count,
|
||||
i64::try_from(TEST_MIGRATOR.iter().count())
|
||||
.expect("migration count fits PostgreSQL bigint")
|
||||
);
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,23 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
num::NonZeroU32,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
model::{PageSize, observability as admin_observability},
|
||||
ports::store::ObservabilityStore as AdminObservabilityStore,
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError,
|
||||
},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
DiagnosticDimension, ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange,
|
||||
ObservabilityRepository, OpsErrorFilter, OpsErrorQuery, PgAdminObservabilityStore,
|
||||
@@ -83,7 +96,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
.expect("persist opaque response IDs");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let records = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
@@ -122,33 +135,32 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability,
|
||||
availability_reason, cooldown_until,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_available', 'openai', 'available', 'user-available', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
$1, $1, $1),
|
||||
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted',
|
||||
$1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
|
||||
null, null, $1, $1, $1)",
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed account metric states");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
@@ -180,6 +192,184 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligible_accounts() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics_cooldowns").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability, quota_limit_reached,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed cooldown account metric states");
|
||||
let cooldowns = StaticCooldowns::new([
|
||||
test_cooldown("acct_metrics_active_cooling", 2),
|
||||
test_cooldown("acct_metrics_cooling", 3),
|
||||
test_cooldown("acct_metrics_expired", 5),
|
||||
test_cooldown("acct_metrics_unknown", 6),
|
||||
test_cooldown("acct_metrics_stale_cooldown", 3),
|
||||
]);
|
||||
let repository =
|
||||
PgObservabilityRepository::new(database.pool.clone(), Some(Arc::new(cooldowns)));
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
let metrics = repository
|
||||
.dashboard_summary(range)
|
||||
.await
|
||||
.expect("dashboard summary")
|
||||
.provider_accounts;
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.rate_limited,
|
||||
metrics.quota_exhausted,
|
||||
metrics.expired,
|
||||
metrics.invalid,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.unavailable,
|
||||
),
|
||||
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.total,
|
||||
metrics.active
|
||||
+ metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
struct StaticCooldowns {
|
||||
cooldowns: BTreeMap<ProviderAccountId, ProviderCooldown>,
|
||||
}
|
||||
|
||||
impl StaticCooldowns {
|
||||
fn new(cooldowns: impl IntoIterator<Item = ProviderCooldown>) -> Self {
|
||||
Self {
|
||||
cooldowns: cooldowns
|
||||
.into_iter()
|
||||
.map(|cooldown| (cooldown.account_id().clone(), cooldown))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for StaticCooldowns {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(self.cooldowns.get(account_id).cloned()) })
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
|
||||
ProviderCooldown::new(
|
||||
ProviderAccountId::new(account_id).expect("test account ID"),
|
||||
CredentialRevision::new(revision).expect("test revision"),
|
||||
SystemTime::now() + Duration::from_secs(60),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
let Some(database) = TestDatabase::create("calculated_usage_billing_facts").await else {
|
||||
@@ -194,7 +384,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
.expect("seed calculated billing facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let facts = repository
|
||||
.usage_calculated_billing_facts(range, UsageRecordFilter::default())
|
||||
@@ -208,7 +398,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
let facts = store
|
||||
.usage_calculated_billing_facts(
|
||||
admin_observability::TimeRange::new(range.start, range.end)
|
||||
@@ -236,7 +426,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
let range =
|
||||
admin_observability::TimeRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("admin observability range");
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
|
||||
let dashboard = store
|
||||
.dashboard_summary(range)
|
||||
@@ -514,6 +704,54 @@ fn admin_page(value: u32) -> admin_observability::PageNumber {
|
||||
admin_observability::PageNumber::new(NonZeroU32::new(value).expect("positive page"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_totals_include_history_outside_selected_range() {
|
||||
let Some(database) = TestDatabase::create("observability_dashboard_totals").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now)
|
||||
.await
|
||||
.expect("seed observability facts");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set started_at = $1 - interval '2 days'
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("move historical request outside selected range");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let dashboard = repository
|
||||
.dashboard_summary(range)
|
||||
.await
|
||||
.expect("dashboard summary");
|
||||
assert_eq!(dashboard.requests.request_count, 2);
|
||||
assert_eq!(
|
||||
(
|
||||
dashboard.totals.request_count,
|
||||
dashboard.totals.input_tokens,
|
||||
dashboard.totals.cached_tokens,
|
||||
dashboard.totals.total_tokens,
|
||||
),
|
||||
(3, 100, 40, 120),
|
||||
);
|
||||
assert_eq!(
|
||||
dashboard
|
||||
.totals
|
||||
.billing_usd
|
||||
.as_ref()
|
||||
.map(|amount| amount.as_str()),
|
||||
Some("1.25"),
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observability_queries_preserve_request_account_cost_and_diagnostic_facts() {
|
||||
let Some(database) = TestDatabase::create("observability").await else {
|
||||
@@ -525,7 +763,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("seed observability facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let dashboard = repository
|
||||
.dashboard_summary(range)
|
||||
|
||||
@@ -28,9 +28,9 @@ use gateway_store::{
|
||||
postgres::{
|
||||
AdminAuditActorKind, AdminAuditEvent, DeleteProviderAccounts, ImportProviderAccounts,
|
||||
NewProviderAccount, PgAdminAccountStore, PgProviderAccountRepository,
|
||||
ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountObservation,
|
||||
ProviderAccountRepository, ProviderCredentialUpdate, RotateProviderAccount,
|
||||
SetProviderAccountEnabled, UpdateProviderAccount,
|
||||
ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountRepository,
|
||||
ProviderCredentialUpdate, RotateProviderAccount, SetProviderAccountEnabled,
|
||||
UpdateProviderAccount,
|
||||
},
|
||||
};
|
||||
use serde_json::json;
|
||||
@@ -49,6 +49,35 @@ fn postgres_provider_account_adapter_implements_core_port() {
|
||||
assert_terminal_admin_port::<PgAdminAccountStore>();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_account_should_allow_missing_upstream_user_id() {
|
||||
let Some(database) = TestDatabase::create("provider_account_missing_upstream_identity").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let mut pending = account("acct_pending_identity", "unused");
|
||||
pending.upstream_user_id = None;
|
||||
pending.email = None;
|
||||
pending.availability = AccountAvailability::Ready;
|
||||
repository
|
||||
.insert_provider_account(pending)
|
||||
.await
|
||||
.expect("insert account without upstream identity");
|
||||
|
||||
let stored = repository
|
||||
.load_provider_account("acct_pending_identity")
|
||||
.await
|
||||
.expect("load account")
|
||||
.expect("stored account");
|
||||
assert_eq!(
|
||||
(stored.summary.upstream_user_id, stored.summary.availability,),
|
||||
(None, AccountAvailability::Unknown)
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
|
||||
let Some(database) = TestDatabase::create("provider_account_quota_batch").await else {
|
||||
@@ -74,6 +103,7 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -140,6 +170,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
};
|
||||
assert_eq!(
|
||||
repository
|
||||
@@ -158,11 +189,10 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: Some(newer),
|
||||
observed_at: newer,
|
||||
})
|
||||
.await
|
||||
@@ -170,11 +200,10 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
assert!(
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: older,
|
||||
})
|
||||
.await
|
||||
@@ -202,6 +231,61 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let account_id = ProviderAccountId::new("acct_state_message").expect("account ID");
|
||||
repository
|
||||
.insert_provider_account(account(account_id.as_str(), "user-state-message"))
|
||||
.await
|
||||
.expect("insert state message fixture");
|
||||
let revision = CredentialRevision::new(1).expect("revision");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: Some("upstream returned 402 payment required".to_owned()),
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist error message");
|
||||
let stored: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read error message");
|
||||
assert_eq!(
|
||||
stored.as_deref(),
|
||||
Some("upstream returned 402 payment required")
|
||||
);
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("recover account");
|
||||
let cleared: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read cleared error message");
|
||||
assert_eq!(cleared, None);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_usage() {
|
||||
let Some(database) = TestDatabase::create("provider_account_terminal_list").await else {
|
||||
@@ -233,6 +317,21 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("insert account list fixture");
|
||||
}
|
||||
repository
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({"rate_limit": {"limit_reached": true}})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed rate-limited account");
|
||||
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
@@ -291,7 +390,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("seed charlie usage");
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let usage_page = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
@@ -309,14 +408,18 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(usage_page.config_revision.get(), 1);
|
||||
assert_eq!(usage_page.total, 6);
|
||||
assert_eq!(usage_page.summary.total, 6);
|
||||
assert_eq!(usage_page.summary.active, 2);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 1);
|
||||
assert_eq!(usage_page.summary.unavailable, 3);
|
||||
assert_eq!(usage_page.summary.normal, 1);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 2);
|
||||
assert_eq!(usage_page.summary.rate_limited, 0);
|
||||
assert_eq!(usage_page.summary.disabled, 1);
|
||||
assert_eq!(usage_page.summary.error, 2);
|
||||
assert_eq!(
|
||||
usage_page.summary.total,
|
||||
usage_page.summary.active
|
||||
usage_page.summary.normal
|
||||
+ usage_page.summary.quota_exhausted
|
||||
+ usage_page.summary.unavailable
|
||||
+ usage_page.summary.rate_limited
|
||||
+ usage_page.summary.disabled
|
||||
+ usage_page.summary.error
|
||||
);
|
||||
assert_eq!(
|
||||
usage_page
|
||||
@@ -356,7 +459,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("Provider kind")),
|
||||
search: Some("ALPHA@EXAMPLE".to_owned()),
|
||||
status: Some(AccountStatus::Active),
|
||||
status: Some(AccountStatus::Normal),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
@@ -366,18 +469,20 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(filtered.items[0].id, "acct_alpha");
|
||||
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
|
||||
|
||||
let banned = store
|
||||
let error_accounts = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: Some(AccountStatus::Banned),
|
||||
status: Some(AccountStatus::Error),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("filter banned accounts");
|
||||
assert_eq!(banned.items[0].id, "acct_beta");
|
||||
.expect("filter error accounts");
|
||||
assert_eq!(error_accounts.items.len(), 2);
|
||||
assert_eq!(error_accounts.items[0].id, "acct_beta");
|
||||
assert_eq!(error_accounts.items[1].id, "acct_invalid");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -405,7 +510,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.expect("insert xAI credential fixture");
|
||||
}
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let provider = ProviderKind::new("xai").expect("xAI Provider kind");
|
||||
let complete = store
|
||||
.list_credentials(
|
||||
@@ -486,7 +591,7 @@ async fn terminal_admin_usage_chunks_large_selections_and_preserves_exact_costs(
|
||||
let mut account_ids = vec!["acct_usage_exact".to_owned()];
|
||||
account_ids.extend((0..200).map(|index| format!("missing_account_{index}")));
|
||||
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage(
|
||||
TimeRange {
|
||||
start: now - TimeDelta::hours(1),
|
||||
@@ -578,7 +683,7 @@ async fn terminal_admin_quota_window_usage_prefers_provider_total_and_falls_back
|
||||
.await
|
||||
.expect("separate provider total from fallback components");
|
||||
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage_by_windows(&[
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_quota_window".to_owned(),
|
||||
@@ -618,7 +723,7 @@ async fn terminal_admin_mutations_keep_revision_account_and_audit_atomic() {
|
||||
.insert_provider_account(account("acct_terminal_mutation", "user-terminal-mutation"))
|
||||
.await
|
||||
.expect("insert mutation account");
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "request_terminal_mutation".to_owned(),
|
||||
@@ -691,7 +796,7 @@ async fn terminal_admin_delete_removes_enabled_accounts_in_one_transaction() {
|
||||
.expect("insert enabled account");
|
||||
}
|
||||
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone())
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: vec![
|
||||
@@ -747,7 +852,6 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
updated.name = "updated import".to_owned();
|
||||
updated.provider_credentials_json = credential_json("updated-import-secret");
|
||||
updated.availability = AccountAvailability::Banned;
|
||||
updated.availability_reason = Some("upstream_account_deactivated".to_owned());
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -765,10 +869,9 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
i64,
|
||||
Option<serde_json::Value>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select name, provider_credentials_json, credential_revision, provider_quota_json,
|
||||
availability, availability_reason
|
||||
availability
|
||||
from provider_accounts where id = 'acct_admin_upsert'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
@@ -782,7 +885,6 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
"credential replacement must clear stale quota"
|
||||
);
|
||||
assert_eq!(row.4, "banned");
|
||||
assert_eq!(row.5.as_deref(), Some("upstream_account_deactivated"));
|
||||
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
@@ -809,7 +911,7 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
|
||||
provider_kind: "xai".to_owned(),
|
||||
name: "before refresh".to_owned(),
|
||||
email: Some("before@example.invalid".to_owned()),
|
||||
upstream_user_id: "upstream-core-refresh".to_owned(),
|
||||
upstream_user_id: Some("upstream-core-refresh".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("free".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -819,8 +921,6 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -942,8 +1042,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
};
|
||||
|
||||
let mut cooldown_account = account("acct_admin_b", "user-admin-b");
|
||||
cooldown_account.availability = AccountAvailability::Cooldown;
|
||||
cooldown_account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(10));
|
||||
cooldown_account.availability = AccountAvailability::Ready;
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -953,24 +1052,21 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.await
|
||||
.expect("import provider accounts");
|
||||
assert_eq!(imported.config_revision.get(), 2);
|
||||
let ready: (bool, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
|
||||
"select enabled, availability, cooldown_until from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load ready imported account");
|
||||
assert_eq!(ready, (true, "ready".to_owned(), None));
|
||||
let cooldown: (bool, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
|
||||
"select enabled, availability, cooldown_until from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_b")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load cooldown imported account");
|
||||
let ready: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load ready imported account");
|
||||
assert_eq!(ready, (true, "ready".to_owned()));
|
||||
let cooldown: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
.bind("acct_admin_b")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load cooldown imported account");
|
||||
assert!(cooldown.0);
|
||||
assert_eq!(cooldown.1, "cooldown");
|
||||
assert!(cooldown.2.is_some());
|
||||
assert_eq!(cooldown.1, "ready");
|
||||
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
@@ -1017,7 +1113,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = 'expired', availability_reason = 'credential_expired'
|
||||
set availability = 'expired'
|
||||
where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1040,15 +1136,8 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.expect("rotate provider account");
|
||||
assert_eq!(rotation.config_revision.get(), 3);
|
||||
assert_eq!(rotation.credential_revision.get(), 2);
|
||||
let restored: (
|
||||
String,
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select availability, availability_reason, cooldown_until,
|
||||
upstream_user_id, upstream_account_id
|
||||
let restored: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select availability, upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1059,8 +1148,6 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
restored,
|
||||
(
|
||||
"ready".to_owned(),
|
||||
None,
|
||||
None,
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)
|
||||
@@ -1158,7 +1245,6 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
};
|
||||
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
|
||||
exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
exhausted.availability_reason = Some("quota_exhausted".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -1187,20 +1273,13 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
.await
|
||||
.expect("rotate exhausted account credential");
|
||||
|
||||
let availability: (String, Option<String>) = sqlx::query_as(
|
||||
"select availability, availability_reason from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(
|
||||
availability,
|
||||
(
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
let availability: String =
|
||||
sqlx::query_scalar("select availability from provider_accounts where id = $1")
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(availability, "quota_exhausted".to_owned());
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1221,32 +1300,24 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("upstream_free_quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI free quota exhaustion");
|
||||
|
||||
let current: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI quota state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("upstream_free_quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1266,38 +1337,26 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
|
||||
.expect("insert xAI account");
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
let cooldown_until = observed_at
|
||||
.checked_add(Duration::from_secs(24 * 60 * 60))
|
||||
.expect("cooldown deadline");
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(cooldown_until),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI resettable usage limit");
|
||||
|
||||
let current: (String, String, Option<String>, bool) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason, cooldown_until is not null
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI usage limit state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"cooldown".to_owned(),
|
||||
Some("usage_limit_exhausted".to_owned()),
|
||||
true,
|
||||
),
|
||||
);
|
||||
assert_eq!(current, ("xai".to_owned(), "ready".to_owned()));
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1315,7 +1374,6 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
|
||||
disabled.enabled = false;
|
||||
disabled.availability = AccountAvailability::Expired;
|
||||
disabled.availability_reason = Some("credential_expired".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -1331,11 +1389,10 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1355,9 +1412,8 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
.await
|
||||
.expect("refresh disabled account credential");
|
||||
|
||||
let current: (bool, String, Option<String>, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, availability_reason,
|
||||
provider_credentials_json, credential_revision
|
||||
let current: (bool, String, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, provider_credentials_json, credential_revision
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
@@ -1366,9 +1422,8 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
.expect("load disabled account after refresh writes");
|
||||
assert!(!current.0);
|
||||
assert_eq!(current.1, "expired");
|
||||
assert_eq!(current.2.as_deref(), Some("credential_expired"));
|
||||
assert_eq!(current.3["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.4, 2);
|
||||
assert_eq!(current.2["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.3, 2);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1379,7 +1434,7 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
provider_kind: "openai".to_owned(),
|
||||
name: id.to_owned(),
|
||||
email: Some(format!("{id}@example.invalid")),
|
||||
upstream_user_id: upstream_user_id.to_owned(),
|
||||
upstream_user_id: Some(upstream_user_id.to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("pro".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1389,8 +1444,6 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
@@ -1517,31 +1570,15 @@ fn provider_credentials_are_redacted_from_debug() {
|
||||
assert!(!format!("{object:?}").contains(secret));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_observation_requires_expiry() {
|
||||
let observation = ProviderAccountObservation {
|
||||
account_id: "account-1".to_owned(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
provider_quota_json: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
quota_observed_at: None,
|
||||
};
|
||||
assert!(observation.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
|
||||
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
|
||||
imported.availability = AccountAvailability::QuotaExhausted;
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_cooldown_until_for_ready_runtime_facts() {
|
||||
let mut imported = account("acct_invalid_cooldown", "user-invalid-cooldown");
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_err());
|
||||
fn imported_ready_account_passes_validation() {
|
||||
let imported = account("acct_ready_import", "user-ready-import");
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ fn runtime_settings_reject_invalid_model_mapping() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_margin_change_should_reschedule_existing_refreshable_accounts() {
|
||||
async fn refresh_margin_change_should_preserve_existing_account_refresh_facts() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_reschedule").await else {
|
||||
return;
|
||||
};
|
||||
@@ -55,6 +55,7 @@ async fn refresh_margin_change_should_reschedule_existing_refreshable_accounts()
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = account_refresh_facts(&database.pool, "acct_refresh_margin_changed").await;
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(1_800))
|
||||
@@ -62,14 +63,14 @@ async fn refresh_margin_change_should_reschedule_existing_refreshable_accounts()
|
||||
.expect("update refresh margin");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_changed").await,
|
||||
expires_at - TimeDelta::seconds(1_800)
|
||||
account_refresh_facts(&database.pool, "acct_refresh_margin_changed").await,
|
||||
before
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unchanged_refresh_margin_should_preserve_existing_retry_schedule() {
|
||||
async fn unchanged_refresh_margin_should_preserve_existing_account_refresh_facts() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_unchanged").await else {
|
||||
return;
|
||||
};
|
||||
@@ -83,6 +84,7 @@ async fn unchanged_refresh_margin_should_preserve_existing_retry_schedule() {
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = account_refresh_facts(&database.pool, "acct_refresh_margin_unchanged").await;
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(3_600))
|
||||
@@ -90,8 +92,8 @@ async fn unchanged_refresh_margin_should_preserve_existing_retry_schedule() {
|
||||
.expect("update unrelated runtime settings");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_unchanged").await,
|
||||
retry_at
|
||||
account_refresh_facts(&database.pool, "acct_refresh_margin_unchanged").await,
|
||||
before
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -118,12 +120,31 @@ async fn insert_refreshable_account(
|
||||
.expect("insert refreshable account");
|
||||
}
|
||||
|
||||
async fn account_next_refresh_at(pool: &sqlx::PgPool, account_id: &str) -> DateTime<Utc> {
|
||||
sqlx::query_scalar("select next_refresh_at from provider_accounts where id = $1")
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct AccountRefreshFacts {
|
||||
access_token_expires_at: DateTime<Utc>,
|
||||
next_refresh_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
credential_revision: i64,
|
||||
}
|
||||
|
||||
async fn account_refresh_facts(pool: &sqlx::PgPool, account_id: &str) -> AccountRefreshFacts {
|
||||
let (access_token_expires_at, next_refresh_at, updated_at, credential_revision) =
|
||||
sqlx::query_as(
|
||||
"select access_token_expires_at, next_refresh_at, updated_at, credential_revision
|
||||
from provider_accounts
|
||||
where id = $1",
|
||||
)
|
||||
.bind(account_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("load next refresh time")
|
||||
.expect("load account refresh facts");
|
||||
AccountRefreshFacts {
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
updated_at,
|
||||
credential_revision,
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_micros(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
|
||||
@@ -39,7 +39,7 @@ async fn request_snapshots_should_survive_account_deletion() {
|
||||
.await
|
||||
.expect("delete provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
@@ -154,7 +154,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_ab")
|
||||
.await
|
||||
@@ -276,7 +276,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
@@ -376,7 +376,7 @@ async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -445,7 +445,7 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
||||
.expect("insert legacy request without live account");
|
||||
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -511,7 +511,7 @@ async fn failure_diagnostics_should_only_include_errored_requests() {
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -566,7 +566,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
|
||||
.await
|
||||
.expect("set request b latency");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -633,7 +633,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
.await
|
||||
.expect("rename provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
|
||||
@@ -1,27 +1,13 @@
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration as StdDuration, SystemTime},
|
||||
};
|
||||
use std::time::{Duration as StdDuration, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaObservation, QuotaWriteOutcome,
|
||||
},
|
||||
error::{StoreError as CoreStoreError, StoreErrorKind},
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
use gateway_store::{
|
||||
Revision, StoreBackend, StoreError, StoreResult,
|
||||
redis::{
|
||||
CooldownCachingProviderAccountStore, CredentialCooldown, CredentialCooldownRepository,
|
||||
RedisCredentialCooldownRepository,
|
||||
},
|
||||
Revision,
|
||||
redis::{CredentialCooldown, CredentialCooldownRepository, RedisCredentialCooldownRepository},
|
||||
};
|
||||
use redis::aio::ConnectionManager;
|
||||
use uuid::Uuid;
|
||||
@@ -36,409 +22,6 @@ fn credential_cooldown_is_revision_fenced() {
|
||||
assert_eq!(cooldown.credential_revision.get(), 2);
|
||||
}
|
||||
|
||||
struct CooldownTestAccountStore {
|
||||
accounts: Vec<ProviderAccount>,
|
||||
calls: Arc<Mutex<Vec<&'static str>>>,
|
||||
fail_state_write: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for CooldownTestAccountStore {
|
||||
async fn create_account(&self, _account: NewProviderAccount) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not create accounts")
|
||||
}
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
Ok(self
|
||||
.accounts
|
||||
.iter()
|
||||
.find(|candidate| candidate.id() == account)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
Ok(self.accounts.clone())
|
||||
}
|
||||
|
||||
async fn list_for_provider(
|
||||
&self,
|
||||
_provider: &ProviderKind,
|
||||
) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not list one provider")
|
||||
}
|
||||
|
||||
async fn load_credential(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
_expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
_update: CredentialCasUpdate,
|
||||
) -> Result<CredentialCasOutcome, CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
Ok(CredentialCasOutcome::Updated(
|
||||
CredentialRevision::new(8).expect("positive revision"),
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_quotas(
|
||||
&self,
|
||||
_accounts: &[ProviderAccountId],
|
||||
) -> Result<Vec<QuotaObservation>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not read quota")
|
||||
}
|
||||
|
||||
async fn compare_and_swap_quota(
|
||||
&self,
|
||||
_observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not write quota")
|
||||
}
|
||||
|
||||
async fn apply_state_change(&self, _change: AccountStateChange) -> Result<(), CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
if self.fail_state_write {
|
||||
Err(CoreStoreError::new(StoreErrorKind::Unavailable))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_account(&self, _update: ProviderAccountUpdate) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not update account metadata")
|
||||
}
|
||||
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
_enabled: bool,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not change account switches")
|
||||
}
|
||||
|
||||
async fn delete_account(&self, _account: &ProviderAccountId) -> Result<(), CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct CooldownTestCache {
|
||||
calls: Arc<Mutex<Vec<&'static str>>>,
|
||||
cached: Mutex<Vec<CredentialCooldown>>,
|
||||
invalidated: Mutex<Vec<(String, Revision)>>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CredentialCooldownRepository for CooldownTestCache {
|
||||
async fn cache_credential_cooldown(&self, cooldown: &CredentialCooldown) -> StoreResult<bool> {
|
||||
self.calls.lock().expect("calls lock").push("redis");
|
||||
if self.fail {
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::Redis,
|
||||
message: "test cache unavailable".to_owned(),
|
||||
});
|
||||
}
|
||||
self.cached
|
||||
.lock()
|
||||
.expect("cached lock")
|
||||
.push(cooldown.clone());
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn read_credential_cooldown(
|
||||
&self,
|
||||
_provider_account_id: &str,
|
||||
) -> StoreResult<Option<CredentialCooldown>> {
|
||||
unreachable!("write-through adapter does not consume cooldown cache")
|
||||
}
|
||||
|
||||
async fn invalidate_credential_cooldown(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
through_revision: Revision,
|
||||
) -> StoreResult<bool> {
|
||||
self.calls.lock().expect("calls lock").push("redis");
|
||||
if self.fail {
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::Redis,
|
||||
message: "test cache unavailable".to_owned(),
|
||||
});
|
||||
}
|
||||
self.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.push((provider_account_id.to_owned(), through_revision));
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn composition_fixture(
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
composition_fixture_with_accounts(
|
||||
vec![ready_test_account("acct_cooldown_composition")],
|
||||
fail_state_write,
|
||||
fail_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn composition_fixture_with_accounts(
|
||||
accounts: Vec<ProviderAccount>,
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts,
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write,
|
||||
});
|
||||
let cache = Arc::new(CooldownTestCache {
|
||||
calls: Arc::clone(&calls),
|
||||
cached: Mutex::new(Vec::new()),
|
||||
invalidated: Mutex::new(Vec::new()),
|
||||
fail: fail_cache,
|
||||
});
|
||||
let store = CooldownCachingProviderAccountStore::new(
|
||||
authoritative,
|
||||
Arc::clone(&cache) as Arc<dyn CredentialCooldownRepository>,
|
||||
);
|
||||
(store, cache, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_writes_cache_only_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("persist cooldown");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
assert_eq!(cache.cached.lock().expect("cached lock").len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_does_not_cache_failed_postgres_state() {
|
||||
let (store, cache, calls) = composition_fixture(true, false);
|
||||
|
||||
assert!(
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_keeps_postgres_terminal_when_redis_fails() {
|
||||
let (store, _cache, calls) = composition_fixture(false, true);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("Redis is only a cache");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_state_write_does_not_change_runtime_cooldown() {
|
||||
let disabled = test_provider_account("acct_cooldown_composition").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
);
|
||||
let (store, cache, calls) = composition_fixture_with_accounts(vec![disabled], false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
assert!(
|
||||
cache
|
||||
.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_state_invalidates_same_revision_cooldown_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
let mut change = cooldown_state_change(60);
|
||||
change.availability = AccountAvailability::Ready;
|
||||
change.reason = None;
|
||||
change.cooldown_until = None;
|
||||
|
||||
store
|
||||
.apply_state_change(change)
|
||||
.await
|
||||
.expect("persist ready state");
|
||||
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0].1,
|
||||
Revision::new(7).expect("positive revision")
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_rotation_invalidates_cooldown_through_new_revision() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
|
||||
let outcome = store
|
||||
.compare_and_swap_credential(credential_update())
|
||||
.await
|
||||
.expect("rotate credential");
|
||||
|
||||
assert_eq!(
|
||||
outcome,
|
||||
CredentialCasOutcome::Updated(CredentialRevision::new(8).expect("positive revision"))
|
||||
);
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0].1,
|
||||
Revision::new(8).expect("positive revision")
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_delete_invalidates_all_stale_cooldown_revisions() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
let account_id = ProviderAccountId::new("acct_cooldown_delete").expect("valid account ID");
|
||||
|
||||
store
|
||||
.delete_account(&account_id)
|
||||
.await
|
||||
.expect("delete account");
|
||||
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0],
|
||||
(
|
||||
account_id.to_string(),
|
||||
Revision::new(u64::MAX).expect("positive revision")
|
||||
)
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_startup_hydrates_only_active_postgres_cooldowns() {
|
||||
let now = SystemTime::now();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts: vec![
|
||||
cooldown_test_account("acct_hydrate_active", now + StdDuration::from_secs(60)),
|
||||
cooldown_test_account("acct_hydrate_expired", now - StdDuration::from_secs(1)),
|
||||
ready_test_account("acct_hydrate_ready"),
|
||||
],
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write: false,
|
||||
});
|
||||
let cache = Arc::new(CooldownTestCache {
|
||||
calls,
|
||||
cached: Mutex::new(Vec::new()),
|
||||
invalidated: Mutex::new(Vec::new()),
|
||||
fail: false,
|
||||
});
|
||||
let store = CooldownCachingProviderAccountStore::new(
|
||||
authoritative,
|
||||
Arc::clone(&cache) as Arc<dyn CredentialCooldownRepository>,
|
||||
);
|
||||
|
||||
assert_eq!(store.hydrate(now).await, 1);
|
||||
assert_eq!(
|
||||
cache.cached.lock().expect("cached lock")[0].provider_account_id,
|
||||
"acct_hydrate_active"
|
||||
);
|
||||
}
|
||||
|
||||
fn cooldown_state_change(seconds: u64) -> AccountStateChange {
|
||||
AccountStateChange {
|
||||
account_id: ProviderAccountId::new("acct_cooldown_composition").expect("valid account ID"),
|
||||
expected_revision: CredentialRevision::new(7).expect("positive revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: Some(SystemTime::now() + StdDuration::from_secs(seconds)),
|
||||
observed_at: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn credential_update() -> CredentialCasUpdate {
|
||||
let account_id = ProviderAccountId::new("acct_cooldown_rotation").expect("valid account ID");
|
||||
CredentialCasUpdate::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(7).expect("positive revision"),
|
||||
ProviderAccountUpdate {
|
||||
account_id,
|
||||
name: "Rotated cooldown account".to_owned(),
|
||||
email: None,
|
||||
plan_type: None,
|
||||
},
|
||||
PlaintextCredential::new(serde_json::Map::new()),
|
||||
false,
|
||||
Some(SystemTime::now() + StdDuration::from_secs(3_600)),
|
||||
None,
|
||||
)
|
||||
.expect("valid credential update")
|
||||
}
|
||||
|
||||
fn cooldown_test_account(account_id: &str, cooldown_until: SystemTime) -> ProviderAccount {
|
||||
test_provider_account(account_id).with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::Cooldown,
|
||||
Some(cooldown_until),
|
||||
)
|
||||
}
|
||||
|
||||
fn ready_test_account(account_id: &str) -> ProviderAccount {
|
||||
test_provider_account(account_id).with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
}
|
||||
|
||||
fn test_provider_account(account_id: &str) -> ProviderAccount {
|
||||
ProviderAccount::new(
|
||||
ProviderAccountId::new(account_id).expect("valid account ID"),
|
||||
ProviderKind::new("openai").expect("valid provider kind"),
|
||||
"Cooldown test".to_owned(),
|
||||
"upstream-user".to_owned(),
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(7).expect("positive revision"),
|
||||
Some(SystemTime::now() + StdDuration::from_secs(3_600)),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_cooldown_round_trips_without_raw_account_id_in_key() {
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
use std::io;
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::backup::{BackupObjectMetadata, BackupStorageConfig};
|
||||
use gateway_admin::ports::backup::{BackupObjectStorePort, UploadObjectRequest};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
use gateway_store::backup::s3::S3ObjectStoreAdapter;
|
||||
use secrecy::SecretString;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
const PART_SIZE_ABOVE_TOKIO_DEFAULT: usize = 2 * 1024 * 1024 + 1;
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_file_sends_full_part_beyond_tokio_default_buffer_limit() {
|
||||
let directory = tempfile::tempdir().expect("create temporary directory");
|
||||
let source = directory.path().join("archive.dump");
|
||||
let expected = vec![b'x'; PART_SIZE_ABOVE_TOKIO_DEFAULT];
|
||||
tokio::fs::write(&source, &expected)
|
||||
.await
|
||||
.expect("write temporary archive");
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind multipart probe server");
|
||||
let endpoint = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("read listener address")
|
||||
);
|
||||
let server = tokio::spawn(serve_multipart_probe(listener));
|
||||
let storage = BackupStorageConfig {
|
||||
storage_revision: 1,
|
||||
endpoint,
|
||||
region: "auto".to_owned(),
|
||||
bucket: "backup-bucket".to_owned(),
|
||||
access_key_id: "test-access-key".to_owned(),
|
||||
secret_access_key: SecretString::from("test-secret-key"),
|
||||
prefix: "backups".to_owned(),
|
||||
force_path_style: true,
|
||||
};
|
||||
let request = UploadObjectRequest {
|
||||
object_key: "backups/archive.dump".to_owned(),
|
||||
source,
|
||||
metadata: BackupObjectMetadata::new(
|
||||
"backup_test".to_owned(),
|
||||
"a".repeat(64),
|
||||
Utc::now(),
|
||||
PART_SIZE_ABOVE_TOKIO_DEFAULT as u64,
|
||||
),
|
||||
cancellation: CancellationToken::new(),
|
||||
};
|
||||
|
||||
S3ObjectStoreAdapter::new()
|
||||
.upload_file(&storage, request)
|
||||
.await
|
||||
.expect("upload multipart archive");
|
||||
let uploaded_len = server
|
||||
.await
|
||||
.expect("multipart probe task did not panic")
|
||||
.expect("serve multipart probe");
|
||||
|
||||
assert_eq!(uploaded_len, PART_SIZE_ABOVE_TOKIO_DEFAULT);
|
||||
}
|
||||
|
||||
async fn serve_multipart_probe(listener: TcpListener) -> io::Result<usize> {
|
||||
let mut uploaded_len = None;
|
||||
for _ in 0..3 {
|
||||
let (mut stream, _) = listener.accept().await?;
|
||||
let (request, body) = read_http_request(&mut stream).await?;
|
||||
let response = if request.starts_with("POST ") && request.contains("uploads") {
|
||||
xml_response(
|
||||
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><UploadId>probe-upload</UploadId></InitiateMultipartUploadResult>",
|
||||
)
|
||||
} else if request.starts_with("PUT ") && request.contains("partNumber=") {
|
||||
uploaded_len = Some(body.len());
|
||||
"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\netag: \"probe-part\"\r\n\r\n".to_owned()
|
||||
} else if request.starts_with("POST ") && request.contains("uploadId=") {
|
||||
xml_response(
|
||||
"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"probe-part\"</ETag></CompleteMultipartUploadResult>",
|
||||
)
|
||||
} else {
|
||||
return Err(io::Error::other("unexpected multipart probe request"));
|
||||
};
|
||||
stream.write_all(response.as_bytes()).await?;
|
||||
stream.shutdown().await?;
|
||||
}
|
||||
uploaded_len.ok_or_else(|| io::Error::other("multipart probe did not receive an upload part"))
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) -> io::Result<(String, Vec<u8>)> {
|
||||
let mut bytes = Vec::new();
|
||||
let header_end = loop {
|
||||
if let Some(position) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break position + 4;
|
||||
}
|
||||
let mut chunk = [0_u8; 8 * 1024];
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"multipart probe request ended before headers",
|
||||
));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
};
|
||||
let headers = String::from_utf8_lossy(&bytes[..header_end]).into_owned();
|
||||
let content_len = headers
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.and_then(|(_, value)| value.trim().parse::<usize>().ok())
|
||||
.unwrap_or_default();
|
||||
if headers
|
||||
.to_ascii_lowercase()
|
||||
.contains("expect: 100-continue")
|
||||
{
|
||||
stream.write_all(b"HTTP/1.1 100 Continue\r\n\r\n").await?;
|
||||
}
|
||||
while bytes.len() < header_end + content_len {
|
||||
let mut chunk = [0_u8; 8 * 1024];
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"multipart probe request ended before body",
|
||||
));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
Ok((
|
||||
headers,
|
||||
bytes[header_end..header_end + content_len].to_vec(),
|
||||
))
|
||||
}
|
||||
|
||||
fn xml_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
@@ -27,7 +27,6 @@ hex = "0.4.3"
|
||||
# reqwest 0.12.28 + ring)。rustls 0.23.37 起 ClientHello 扩展顺序变化,JA3
|
||||
# 会偏离官方指纹;升级前必须重新抓握手对比扩展顺序。
|
||||
hyper-rustls = "=0.27.7"
|
||||
jsonwebtoken = { version = "=11.0.0", default-features = false, features = ["aws_lc_rs", "use_pem"] }
|
||||
# 与官方 Codex Desktop 一致:reqwest 不启用压缩特性,上游请求不带 accept-encoding。
|
||||
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "http2"] }
|
||||
roxmltree = "0.21.1"
|
||||
@@ -58,6 +57,7 @@ zstd = "0.13"
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-thread", "test-util"] }
|
||||
tracing-subscriber = { version = "0.3.23", features = ["json"] }
|
||||
wiremock = "0.6.5"
|
||||
|
||||
[[test]]
|
||||
|
||||
@@ -19,7 +19,7 @@ use gateway_admin::model::provider_credentials::{
|
||||
PreparedCredentialCreate, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
PreparedCredentialRotationFacts, ProviderDocument, ProviderExport,
|
||||
ProviderExportCredentialInput, ProviderModel, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
};
|
||||
use gateway_admin::model::{MutationActor, MutationContext, Revision};
|
||||
use gateway_admin::ports::provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind};
|
||||
@@ -33,28 +33,24 @@ use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_core::provider_ports::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome,
|
||||
ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome, ProviderStoreError,
|
||||
ProviderStoreErrorKind,
|
||||
};
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Number, Value};
|
||||
|
||||
use crate::credential::{
|
||||
CodexAccountIdentityVerifier, CodexCredentialCodec, CodexIdentityExpectation, CodexOAuthSecret,
|
||||
oauth_owner_ref,
|
||||
};
|
||||
use crate::credential::{
|
||||
CodexAccountQuotaSnapshot, CodexCredentialAdmin, CodexCredentialAdminError,
|
||||
CodexCredentialAdminService, CodexCredentialCatalogError, CodexCredentialCatalogService,
|
||||
CodexCredentialQuotaError, CodexCredentialQuotaService, CodexOAuthAdmin, CodexOAuthAdminError,
|
||||
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
|
||||
CodexPendingAuthorization, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CodexPendingAuthorization, CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential, ExportManagedCodexCredential,
|
||||
RotateManagedCodexCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
refresh_time,
|
||||
StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
};
|
||||
use crate::credential::{CodexCredentialCodec, CodexOAuthSecret, oauth_owner_ref};
|
||||
use crate::transport::CodexWebSocketPool;
|
||||
use crate::transport::openai_billing_breakdown;
|
||||
use crate::transport::profile::{
|
||||
@@ -63,8 +59,6 @@ use crate::transport::profile::{
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const PENDING_DOCUMENT_SCHEMA_VERSION: u64 = 3;
|
||||
const MAX_ACCESS_TOKEN_BYTES: usize = 16 * 1024;
|
||||
const MAX_REFRESH_TOKEN_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// OpenAI 对终态 Admin port 的唯一实现。
|
||||
pub(crate) struct OpenAiAdminProvider {
|
||||
@@ -72,22 +66,18 @@ pub(crate) struct OpenAiAdminProvider {
|
||||
profile: CodexWireProfileState,
|
||||
accounts: Arc<dyn ProviderAccountStore>,
|
||||
credentials: Arc<CodexCredentialAdminService>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
oauth: Arc<dyn CodexOAuthAdmin>,
|
||||
quota: Arc<CodexCredentialQuotaService>,
|
||||
catalog: Arc<CodexCredentialCatalogService>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
websocket_pool: Arc<CodexWebSocketPool>,
|
||||
desktop_release: CodexDesktopReleaseStatus,
|
||||
}
|
||||
|
||||
pub(crate) struct OpenAiAdminServices {
|
||||
pub(crate) credentials: Arc<CodexCredentialAdminService>,
|
||||
pub(crate) verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
pub(crate) oauth: Arc<dyn CodexOAuthAdmin>,
|
||||
pub(crate) quota: Arc<CodexCredentialQuotaService>,
|
||||
pub(crate) catalog: Arc<CodexCredentialCatalogService>,
|
||||
pub(crate) runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
}
|
||||
|
||||
impl OpenAiAdminProvider {
|
||||
@@ -105,11 +95,9 @@ impl OpenAiAdminProvider {
|
||||
profile,
|
||||
accounts,
|
||||
credentials: services.credentials,
|
||||
verifier: services.verifier,
|
||||
oauth: services.oauth,
|
||||
quota: services.quota,
|
||||
catalog: services.catalog,
|
||||
runtime_policy: services.runtime_policy,
|
||||
websocket_pool,
|
||||
desktop_release,
|
||||
}
|
||||
@@ -194,12 +182,13 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
let profile = self.profile.snapshot();
|
||||
let release = self.desktop_release.snapshot();
|
||||
let user_agent = profile.user_agent();
|
||||
let client_identity = format!("{}; {}", profile.originator, profile.desktop_version);
|
||||
let release = dashboard_desktop_release(&profile, release);
|
||||
Some(DashboardWireProfile {
|
||||
provider: self.provider_kind.as_str().to_owned(),
|
||||
product: profile.originator,
|
||||
version: profile.desktop_version,
|
||||
build: Some(profile.desktop_build),
|
||||
version: profile.codex_version,
|
||||
build: None,
|
||||
target: DashboardWireTarget {
|
||||
os_type: profile.os_type,
|
||||
os_version: profile.os_version,
|
||||
@@ -208,8 +197,8 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
},
|
||||
user_agent,
|
||||
attributes: vec![DashboardWireAttribute {
|
||||
label: "Codex Core".to_owned(),
|
||||
value: profile.codex_version,
|
||||
label: "客户端标识".to_owned(),
|
||||
value: client_identity,
|
||||
}],
|
||||
verified_at: Some(profile.verified_at),
|
||||
release: Some(release),
|
||||
@@ -380,49 +369,9 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
if !account_matches_record(¤t.account, &command.account) {
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Conflict));
|
||||
}
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let account_id = current
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or_else(|| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let principal = runtime
|
||||
.principal
|
||||
.ok_or_else(|| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let expectation = CodexIdentityExpectation::current(
|
||||
principal.oauth_subject,
|
||||
principal.poid,
|
||||
account_id.to_owned(),
|
||||
current.account.upstream_user_id().to_owned(),
|
||||
runtime.installation_id,
|
||||
)
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let secret = rotation_secret(command.provider_material)?;
|
||||
let verified_account = self
|
||||
.verifier
|
||||
.verify(&secret, &expectation)
|
||||
.await
|
||||
.and_then(crate::credential::CodexIdentityVerification::into_complete)
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let policy = self
|
||||
.runtime_policy
|
||||
.load_refresh_policy()
|
||||
.await
|
||||
.map_err(map_provider_store_error)?;
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
current.account.id(),
|
||||
verified_account.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)
|
||||
.map_err(map_credential_admin_error)?;
|
||||
let prepared = CodexCredentialAdmin
|
||||
.prepare_rotation(RotateManagedCodexCredential {
|
||||
current,
|
||||
secret,
|
||||
verified_account,
|
||||
next_refresh_at,
|
||||
})
|
||||
.prepare_refreshed_oauth_rotation(current, secret, None, None)
|
||||
.map_err(map_credential_admin_error)?;
|
||||
prepared_rotation(prepared, command.account.provider_kind)
|
||||
}
|
||||
@@ -475,7 +424,14 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
};
|
||||
Ok(project_quota(snapshot, &account))
|
||||
let mut quota = project_quota(snapshot, &account);
|
||||
quota.rate_limited_until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok(quota)
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -569,7 +525,7 @@ fn prepared_create(
|
||||
provider_kind: account.provider().clone(),
|
||||
name: account.name().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
upstream_user_id: account.upstream_user_id().to_owned(),
|
||||
upstream_user_id: account.upstream_user_id().map(str::to_owned),
|
||||
upstream_account_id: account.upstream_account_id().map(str::to_owned),
|
||||
plan_type: account.plan_type().map(str::to_owned),
|
||||
authentication_kind: account.authentication_kind().to_owned(),
|
||||
@@ -579,8 +535,6 @@ fn prepared_create(
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability,
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: observed_at,
|
||||
})
|
||||
}
|
||||
@@ -635,15 +589,6 @@ fn rotation_secret(document: ProviderDocument) -> Result<CodexOAuthSecret, Provi
|
||||
let document: RotationDocument =
|
||||
serde_json::from_value(Value::Object(document.into_provider_data().into_inner()))
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
if document.access_token.is_empty()
|
||||
|| document.access_token.len() > MAX_ACCESS_TOKEN_BYTES
|
||||
|| document
|
||||
.refresh_token
|
||||
.as_deref()
|
||||
.is_some_and(|token| token.is_empty() || token.len() > MAX_REFRESH_TOKEN_BYTES)
|
||||
{
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Invalid));
|
||||
}
|
||||
Ok(CodexOAuthSecret {
|
||||
access_token: SecretString::from(document.access_token),
|
||||
refresh_token: document.refresh_token.map(SecretString::from),
|
||||
@@ -682,11 +627,7 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
|
||||
account.upstream_account_id.clone(),
|
||||
account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(
|
||||
account.enabled,
|
||||
account.availability,
|
||||
account.cooldown_until.map(SystemTime::from),
|
||||
)
|
||||
.with_runtime_state(account.enabled, account.availability)
|
||||
.with_refresh_schedule(
|
||||
account.has_refresh_token,
|
||||
account.next_refresh_at.map(SystemTime::from),
|
||||
@@ -735,7 +676,7 @@ fn dashboard_desktop_release(
|
||||
fn account_matches_record(account: &ProviderAccount, record: &AccountRecord) -> bool {
|
||||
account.id().as_str() == record.id
|
||||
&& account.provider() == &record.provider_kind
|
||||
&& account.upstream_user_id() == record.upstream_user_id
|
||||
&& account.upstream_user_id() == record.upstream_user_id.as_deref()
|
||||
&& account.upstream_account_id() == record.upstream_account_id.as_deref()
|
||||
&& account.authentication_kind() == record.authentication_kind
|
||||
}
|
||||
@@ -745,6 +686,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -753,9 +696,12 @@ fn project_quota(
|
||||
snapshot: Option<CodexAccountQuotaSnapshot>,
|
||||
account: &ProviderAccount,
|
||||
) -> ProviderQuota {
|
||||
let mut quota = snapshot.map_or_else(empty_quota, project_quota_snapshot);
|
||||
// 展示前先滚动已过期窗口,避免 reset 后的旧窗口值悬挂。
|
||||
let mut quota = snapshot
|
||||
.map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now())))
|
||||
.unwrap_or_else(empty_quota);
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
force_confirmed_exhaustion_projection(&mut quota, account.cooldown_until());
|
||||
force_confirmed_exhaustion_projection(&mut quota);
|
||||
}
|
||||
quota
|
||||
}
|
||||
@@ -773,9 +719,10 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
"exhausted".to_owned(),
|
||||
Value::Bool(snapshot.fact().exhausted()),
|
||||
);
|
||||
let windows = snapshot
|
||||
let windows: Vec<ProviderQuotaWindow> = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| should_project_quota_window(window))
|
||||
.map(|window| {
|
||||
let mut data = Map::new();
|
||||
data.insert(
|
||||
@@ -790,35 +737,61 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
key: window.key().to_owned(),
|
||||
group: quota_group(window.kind()).to_owned(),
|
||||
label: codex_quota_window_label(
|
||||
window.limit_name(),
|
||||
window.kind(),
|
||||
window.role(),
|
||||
window.source(),
|
||||
window.window_seconds(),
|
||||
),
|
||||
source: Some(window.source().to_owned()),
|
||||
local_usage_attribution: if window.is_account_wide() {
|
||||
QuotaLocalUsageAttribution::AccountWide
|
||||
} else {
|
||||
QuotaLocalUsageAttribution::Unavailable
|
||||
},
|
||||
window_seconds: window.window_seconds(),
|
||||
used_percent: window.used_percent(),
|
||||
reset_at: window.reset_at(),
|
||||
limit_reached: window.limit_reached(),
|
||||
local_usage: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(data))),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// 快照级 limit_reached 只看滚动后的窗口触顶:顶层标记是观测事实,不能
|
||||
// 在窗口全部过期后继续维持限流。
|
||||
let limit_reached = quota_windows_limit_reached(&windows);
|
||||
ProviderQuota {
|
||||
observed_at: Some(DateTime::<Utc>::from(snapshot.observed_at())),
|
||||
refresh_token_expires_at: None,
|
||||
windows,
|
||||
limit_reached,
|
||||
rate_limited_until: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
provider_data,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn force_confirmed_exhaustion_projection(
|
||||
quota: &mut ProviderQuota,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) {
|
||||
// 真实 429 对“当前已耗尽”的证明强于可能延迟结算的 usage 文档。这里只覆盖
|
||||
// Admin 展示投影,原始 Provider JSON 仍作为后续判断窗口是否真正恢复的基线。
|
||||
/// `secondary_window` 有时只是上游的空占位:没有时长、重置时间,也没有用量。
|
||||
/// 它不能被可靠翻译成 5 小时或周额度,因此不应占用账号面板;带有实际事实的
|
||||
/// 次级窗口(时长、重置、非零用量或触顶)仍完整保留。
|
||||
fn should_project_quota_window(window: &CodexQuotaWindow) -> bool {
|
||||
window.role() != CodexQuotaWindowRole::Secondary
|
||||
|| window.window_seconds().is_some()
|
||||
|| window.reset_at().is_some()
|
||||
|| window.used_percent().is_some_and(|used| used > 0.0)
|
||||
|| window.limit_reached()
|
||||
}
|
||||
|
||||
fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
|
||||
windows.iter().any(|window| window.limit_reached)
|
||||
}
|
||||
|
||||
fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) {
|
||||
// 402 已确认耗尽:只覆盖 Admin 展示投影,原始 Provider JSON 仍作为
|
||||
// 后续判断窗口是否真正恢复的基线。
|
||||
quota.limit_reached = true;
|
||||
quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
Map::from_iter([
|
||||
(
|
||||
@@ -828,51 +801,37 @@ fn force_confirmed_exhaustion_projection(
|
||||
("exhausted".to_owned(), Value::Bool(true)),
|
||||
]),
|
||||
)));
|
||||
let reset_at = cooldown_until.map(DateTime::<Utc>::from);
|
||||
if quota.windows.is_empty() {
|
||||
quota.windows.push(ProviderQuotaWindow {
|
||||
key: "confirmed-quota-exhaustion".to_owned(),
|
||||
group: "other".to_owned(),
|
||||
label: "额度".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: None,
|
||||
used_percent: Some(100.0),
|
||||
reset_at,
|
||||
reset_at: None,
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let matched_reset = reset_at.and_then(|reset_at| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, window)| {
|
||||
let distance = window.reset_at.map(|window_reset| {
|
||||
(window_reset.timestamp() - reset_at.timestamp()).unsigned_abs()
|
||||
})?;
|
||||
Some((index, distance))
|
||||
})
|
||||
.min_by_key(|(_, distance)| *distance)
|
||||
.map(|(index, _)| index)
|
||||
});
|
||||
let index = matched_reset
|
||||
.or_else(|| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, left), (_, right)| {
|
||||
left.used_percent
|
||||
.unwrap_or(-1.0)
|
||||
.total_cmp(&right.used_percent.unwrap_or(-1.0))
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
// 选已用比例最高的窗口强制 100%,保留其自身 reset_at。
|
||||
let index = quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, left), (_, right)| {
|
||||
left.used_percent
|
||||
.unwrap_or(-1.0)
|
||||
.total_cmp(&right.used_percent.unwrap_or(-1.0))
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
quota.windows[index].used_percent = Some(100.0);
|
||||
quota.windows[index].limit_reached = true;
|
||||
}
|
||||
|
||||
const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str {
|
||||
@@ -891,54 +850,69 @@ const fn quota_role(role: CodexQuotaWindowRole) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// 按窗口时长显示汉化额度名;非核心桶补上上游桶名称作为前缀。
|
||||
fn codex_quota_window_label(
|
||||
limit_name: Option<&str>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
role: CodexQuotaWindowRole,
|
||||
source: &str,
|
||||
window_seconds: Option<u64>,
|
||||
) -> String {
|
||||
let base = match kind {
|
||||
CodexQuotaWindowKind::Monthly => "月限额".to_owned(),
|
||||
CodexQuotaWindowKind::Weekly => "周限额".to_owned(),
|
||||
CodexQuotaWindowKind::Monthly => "月额度".to_owned(),
|
||||
CodexQuotaWindowKind::Weekly => "周额度".to_owned(),
|
||||
CodexQuotaWindowKind::ShortTerm => {
|
||||
if window_seconds.is_some_and(|seconds| seconds > 86_400) {
|
||||
"周限额".to_owned()
|
||||
"周额度".to_owned()
|
||||
} else {
|
||||
"5小时限额".to_owned()
|
||||
"5小时额度".to_owned()
|
||||
}
|
||||
}
|
||||
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds),
|
||||
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds, role),
|
||||
};
|
||||
if matches!(source, "core" | "codex" | "spend_control" | "monthly_limit") {
|
||||
return base;
|
||||
if let Some(prefix) = codex_quota_window_prefix(limit_name, source) {
|
||||
return format!("{prefix} · {base}");
|
||||
}
|
||||
if is_codex_review_limit(source) {
|
||||
return format!("代码审查 · {base}");
|
||||
}
|
||||
format!("{source} · {base}")
|
||||
base
|
||||
}
|
||||
|
||||
fn custom_quota_window_label(window_seconds: Option<u64>) -> String {
|
||||
fn codex_quota_window_prefix(limit_name: Option<&str>, source: &str) -> Option<String> {
|
||||
let source_identifier = quota_identifier(source);
|
||||
if matches!(source_identifier.as_str(), "core" | "codex") {
|
||||
return None;
|
||||
}
|
||||
let label = limit_name.unwrap_or(source);
|
||||
match quota_identifier(label).as_str() {
|
||||
"premium" => Some("Premium".to_owned()),
|
||||
"code_review" => Some("代码审查".to_owned()),
|
||||
"spend_control" => Some("消费控制".to_owned()),
|
||||
_ => Some(label.replace('_', " ")),
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_identifier(value: &str) -> String {
|
||||
value.trim().to_ascii_lowercase().replace(['-', ' '], "_")
|
||||
}
|
||||
|
||||
fn custom_quota_window_label(window_seconds: Option<u64>, role: CodexQuotaWindowRole) -> String {
|
||||
let Some(seconds) = window_seconds.filter(|seconds| *seconds > 0) else {
|
||||
return "额度".to_owned();
|
||||
// 与官方客户端一致:没有时长时不臆测为 5 小时或周额度;保留
|
||||
// primary/secondary 语义,让用户知道这是上游未标明时长的独立窗口。
|
||||
return match role {
|
||||
CodexQuotaWindowRole::Primary => "主额度".to_owned(),
|
||||
CodexQuotaWindowRole::Secondary => "次级额度".to_owned(),
|
||||
CodexQuotaWindowRole::Monthly => "月额度".to_owned(),
|
||||
};
|
||||
};
|
||||
if seconds % 86_400 == 0 {
|
||||
format!("{}日限额", seconds / 86_400)
|
||||
format!("{}日额度", seconds / 86_400)
|
||||
} else if seconds % 3_600 == 0 {
|
||||
format!("{}小时限额", seconds / 3_600)
|
||||
format!("{}小时额度", seconds / 3_600)
|
||||
} else {
|
||||
format!("{}分钟限额", seconds.div_ceil(60))
|
||||
format!("{}分钟额度", seconds.div_ceil(60))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_codex_review_limit(value: &str) -> bool {
|
||||
let normalized = value.trim().to_ascii_lowercase().replace(['-', ' '], "_");
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"review" | "code_review" | "codex_review" | "codex_code_review"
|
||||
) || normalized.contains("code_review")
|
||||
|| normalized.contains("codex_review")
|
||||
}
|
||||
|
||||
/// 将 Provider-owned PKCE/OIDC 状态保存到 Store 提供的 Redis 原子端口。
|
||||
pub(crate) struct OpenAiOAuthPendingStore {
|
||||
port: Arc<dyn OAuthPendingFlowPort>,
|
||||
@@ -1324,14 +1298,6 @@ fn map_pending_store_error(error: ProviderStoreError) -> CodexOAuthPendingStoreE
|
||||
}
|
||||
}
|
||||
|
||||
fn map_provider_store_error(error: ProviderStoreError) -> ProviderAdminError {
|
||||
provider_admin_error(match error.kind() {
|
||||
ProviderStoreErrorKind::InvalidData => ProviderAdminErrorKind::Invalid,
|
||||
ProviderStoreErrorKind::Conflict => ProviderAdminErrorKind::Conflict,
|
||||
ProviderStoreErrorKind::Unavailable => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
fn map_store_error(error: gateway_core::error::StoreError) -> ProviderAdminError {
|
||||
provider_admin_error(match error.kind() {
|
||||
StoreErrorKind::Conflict => ProviderAdminErrorKind::Conflict,
|
||||
@@ -1348,25 +1314,20 @@ fn map_credential_admin_error(error: CodexCredentialAdminError) -> ProviderAdmin
|
||||
provider_admin_error(match error {
|
||||
Error::InvalidInput
|
||||
| Error::InvalidCredential
|
||||
| Error::IdentityMismatch
|
||||
| Error::MissingRefreshToken
|
||||
| Error::RefreshRejected
|
||||
| Error::AccountBanned
|
||||
| Error::IdentityRejected => ProviderAdminErrorKind::Invalid,
|
||||
| Error::AccountBanned => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound => ProviderAdminErrorKind::NotFound,
|
||||
Error::RefreshLeaseUnavailable | Error::RefreshAmbiguous => {
|
||||
ProviderAdminErrorKind::Conflict
|
||||
}
|
||||
Error::RefreshUnavailable | Error::IdentityUnavailable => {
|
||||
ProviderAdminErrorKind::Unavailable
|
||||
}
|
||||
Error::RefreshUnavailable => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
const fn credential_admin_error_code(error: CodexCredentialAdminError) -> &'static str {
|
||||
match error {
|
||||
CodexCredentialAdminError::InvalidInput => "invalid_input",
|
||||
CodexCredentialAdminError::IdentityMismatch => "identity_mismatch",
|
||||
CodexCredentialAdminError::InvalidCredential => "invalid_credential",
|
||||
CodexCredentialAdminError::NotFound => "not_found",
|
||||
CodexCredentialAdminError::MissingRefreshToken => "missing_refresh_token",
|
||||
@@ -1375,8 +1336,6 @@ const fn credential_admin_error_code(error: CodexCredentialAdminError) -> &'stat
|
||||
CodexCredentialAdminError::AccountBanned => "account_banned",
|
||||
CodexCredentialAdminError::RefreshUnavailable => "refresh_unavailable",
|
||||
CodexCredentialAdminError::RefreshAmbiguous => "refresh_ambiguous",
|
||||
CodexCredentialAdminError::IdentityRejected => "identity_rejected",
|
||||
CodexCredentialAdminError::IdentityUnavailable => "identity_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1405,7 +1364,6 @@ fn map_oauth_error(error: CodexOAuthAdminError) -> ProviderAdminError {
|
||||
Error::InvalidInput
|
||||
| Error::CallbackRejected
|
||||
| Error::TokenRejected
|
||||
| Error::IdentityRejected
|
||||
| Error::Credential => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound | Error::FlowExpired => ProviderAdminErrorKind::NotFound,
|
||||
Error::Conflict | Error::Ambiguous => ProviderAdminErrorKind::Conflict,
|
||||
@@ -1423,7 +1381,6 @@ const fn oauth_error_code(error: &CodexOAuthAdminError) -> &'static str {
|
||||
CodexOAuthAdminError::FlowExpired => "flow_expired",
|
||||
CodexOAuthAdminError::CallbackRejected => "callback_rejected",
|
||||
CodexOAuthAdminError::TokenRejected => "token_rejected",
|
||||
CodexOAuthAdminError::IdentityRejected => "identity_rejected",
|
||||
CodexOAuthAdminError::UpstreamUnavailable => "upstream_unavailable",
|
||||
CodexOAuthAdminError::Ambiguous => "ambiguous",
|
||||
CodexOAuthAdminError::StorageUnavailable => "storage_unavailable",
|
||||
|
||||
@@ -43,10 +43,8 @@ impl OpenAiConfig {
|
||||
self.quota.validate()?;
|
||||
self.auth.validate()?;
|
||||
self.wire_profile.validate()?;
|
||||
self.identity_secret_path = source_dir
|
||||
.parent()
|
||||
.unwrap_or(source_dir)
|
||||
.join(".runtime/data/identity_hmac_secret");
|
||||
let project_root = source_dir.parent().unwrap_or(source_dir);
|
||||
self.identity_secret_path = project_root.join(".runtime/data/identity_hmac_secret");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -278,6 +276,7 @@ impl CodexAuthSettings {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CodexWireProfileConfig {
|
||||
pub originator: String,
|
||||
/// npm `@openai/codex` 启动版本;运行时发布任务会更新此画像字段。
|
||||
pub codex_version: String,
|
||||
pub desktop_version: String,
|
||||
pub desktop_build: String,
|
||||
@@ -292,9 +291,9 @@ impl Default for CodexWireProfileConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
originator: "Codex Desktop".to_owned(),
|
||||
codex_version: "0.145.0".to_owned(),
|
||||
desktop_version: "26.721.31836".to_owned(),
|
||||
desktop_build: "5828".to_owned(),
|
||||
codex_version: "0.146.0".to_owned(),
|
||||
desktop_version: "26.727.51351".to_owned(),
|
||||
desktop_build: "6119".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
|
||||
@@ -16,8 +16,7 @@ use gateway_core::engine::credential::{
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
ProviderRuntimePolicyPort,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRuntimePolicyPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
@@ -25,21 +24,21 @@ use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
};
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{RefreshFailure, TokenRefresher};
|
||||
use super::types::{
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CODEX_AUTHENTICATION_KIND_OAUTH, CodexAccountProfile,
|
||||
CodexAgentIdentityAuthMode, CodexAgentIdentityCredentialData, CodexCredentialData,
|
||||
CodexOAuthSecret,
|
||||
CodexCredentialPrincipal, CodexOAuthMetadata, CodexOAuthSecret, parse_access_token_expiration,
|
||||
};
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const CODEX_CREDENTIAL_SCHEMA_VERSION: u32 = 1;
|
||||
const MAX_BATCH: usize = 200;
|
||||
const MAX_IMPORT_DOCUMENT_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_IMPORT_EMAIL_BYTES: usize = 512;
|
||||
const MAX_IMPORT_PLAN_BYTES: usize = 128;
|
||||
|
||||
pub struct ImportCodexOAuthCredential {
|
||||
pub account_id: String,
|
||||
@@ -50,6 +49,35 @@ pub struct ImportCodexOAuthCredential {
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
/// OAuth credential 的最小创建输入。
|
||||
///
|
||||
/// metadata 来自 ID token 的本地 payload 解码,或导入文档的展示字段;不包含
|
||||
/// 签名、issuer、audience、nonce 或 access/refresh token 验证。
|
||||
pub(crate) struct UnresolvedCodexOAuthCredential {
|
||||
pub(crate) account_id: String,
|
||||
pub(crate) name: String,
|
||||
pub(crate) secret: CodexOAuthSecret,
|
||||
pub(crate) metadata: CodexOAuthMetadata,
|
||||
pub(crate) access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub(crate) next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub(crate) enabled: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for UnresolvedCodexOAuthCredential {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("UnresolvedCodexOAuthCredential")
|
||||
.field("account_id", &self.account_id)
|
||||
.field("name", &self.name)
|
||||
.field("secret", &"<redacted>")
|
||||
.field("metadata", &self.metadata)
|
||||
.field("access_token_expires_at", &self.access_token_expires_at)
|
||||
.field("next_refresh_at", &self.next_refresh_at)
|
||||
.field("enabled", &self.enabled)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ImportCodexOAuthCredential {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
@@ -106,6 +134,7 @@ enum ParsedCodexAuthentication {
|
||||
OAuth {
|
||||
access_token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
id_token: Option<String>,
|
||||
},
|
||||
AgentIdentity {
|
||||
runtime_id: String,
|
||||
@@ -120,6 +149,7 @@ impl fmt::Debug for ParsedCodexAuthentication {
|
||||
Self::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token,
|
||||
} => formatter
|
||||
.debug_struct("OAuth")
|
||||
.field("access_token", &access_token.as_ref().map(|_| "<redacted>"))
|
||||
@@ -127,6 +157,7 @@ impl fmt::Debug for ParsedCodexAuthentication {
|
||||
"refresh_token",
|
||||
&refresh_token.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field("id_token", &id_token.as_ref().map(|_| "<redacted>"))
|
||||
.finish(),
|
||||
Self::AgentIdentity {
|
||||
runtime_id: _,
|
||||
@@ -347,8 +378,6 @@ impl fmt::Debug for PreparedCodexCredentialRotationGuard {
|
||||
pub enum CodexCredentialAdminError {
|
||||
#[error("Codex account input is invalid")]
|
||||
InvalidInput,
|
||||
#[error("Codex account identity does not match the existing account")]
|
||||
IdentityMismatch,
|
||||
#[error("Codex credential JSON is invalid")]
|
||||
InvalidCredential,
|
||||
#[error("Codex account was not found")]
|
||||
@@ -365,10 +394,6 @@ pub enum CodexCredentialAdminError {
|
||||
RefreshUnavailable,
|
||||
#[error("Codex refresh send state is ambiguous")]
|
||||
RefreshAmbiguous,
|
||||
#[error("Codex refreshed identity was rejected")]
|
||||
IdentityRejected,
|
||||
#[error("Codex identity verification is unavailable")]
|
||||
IdentityUnavailable,
|
||||
}
|
||||
|
||||
/// 无状态的 Codex Admin command preparer。
|
||||
@@ -387,8 +412,7 @@ impl CodexCredentialAdmin {
|
||||
if input.name.trim().is_empty() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let access_token_expires_at =
|
||||
required_time(input.verified_account.access_token_expires_at)?;
|
||||
let access_token_expires_at = optional_time(input.verified_account.access_token_expires_at);
|
||||
let revision =
|
||||
CredentialRevision::new(1).map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let upstream_user_id = input.verified_account.chatgpt_user_id.clone();
|
||||
@@ -399,17 +423,70 @@ impl CodexCredentialAdmin {
|
||||
account_id,
|
||||
provider,
|
||||
input.name,
|
||||
upstream_user_id,
|
||||
Some(upstream_user_id),
|
||||
CODEX_AUTHENTICATION_KIND_OAUTH.to_owned(),
|
||||
revision,
|
||||
Some(access_token_expires_at),
|
||||
access_token_expires_at,
|
||||
)
|
||||
.with_profile(
|
||||
input.verified_account.email,
|
||||
Some(input.verified_account.chatgpt_account_id),
|
||||
input.verified_account.plan_type,
|
||||
)
|
||||
.with_runtime_state(input.enabled, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(input.enabled, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(
|
||||
input.secret.refresh_token.is_some(),
|
||||
optional_time(input.next_refresh_at),
|
||||
);
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
credential,
|
||||
})
|
||||
}
|
||||
|
||||
/// 为已取得 OAuth access token、但资料尚未补全的账号创建最小记录。
|
||||
///
|
||||
/// ID token payload 只用于本地字段投影;创建路径绝不发起 usage/profile 请求。
|
||||
pub(crate) fn prepare_unresolved_oauth(
|
||||
&self,
|
||||
input: UnresolvedCodexOAuthCredential,
|
||||
) -> Result<NewProviderAccount, CodexCredentialAdminError> {
|
||||
let account_id = ProviderAccountId::new(input.account_id)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?;
|
||||
let provider = ProviderKind::new(PROVIDER_NAME)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?;
|
||||
if input.name.trim().is_empty() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let revision =
|
||||
CredentialRevision::new(1).map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let CodexOAuthMetadata {
|
||||
email,
|
||||
chatgpt_plan_type: plan_type,
|
||||
chatgpt_user_id: upstream_user_id,
|
||||
chatgpt_account_id: upstream_account_id,
|
||||
} = input.metadata;
|
||||
let has_upstream_user_id = upstream_user_id.is_some();
|
||||
let credential = CodexCredentialCodec::encode_unresolved(&input.secret, Vec::new())
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let account = ProviderAccount::new(
|
||||
account_id,
|
||||
provider,
|
||||
input.name,
|
||||
upstream_user_id,
|
||||
CODEX_AUTHENTICATION_KIND_OAUTH.to_owned(),
|
||||
revision,
|
||||
optional_time(input.access_token_expires_at),
|
||||
)
|
||||
.with_profile(email, upstream_account_id, plan_type)
|
||||
.with_runtime_state(
|
||||
input.enabled,
|
||||
if has_upstream_user_id {
|
||||
AccountAvailability::Ready
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
},
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
input.secret.refresh_token.is_some(),
|
||||
optional_time(input.next_refresh_at),
|
||||
@@ -444,7 +521,7 @@ impl CodexCredentialAdmin {
|
||||
id: account.id().as_str().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
account_id: account.upstream_account_id().map(str::to_owned),
|
||||
user_id: Some(account.upstream_user_id().to_owned()),
|
||||
user_id: account.upstream_user_id().map(str::to_owned),
|
||||
label: Some(account.name().to_owned()),
|
||||
plan_type: account.plan_type().map(str::to_owned),
|
||||
status: cpr_status(&account),
|
||||
@@ -499,12 +576,60 @@ impl CodexCredentialAdmin {
|
||||
self.prepare_oauth_rotation(input, false)
|
||||
}
|
||||
|
||||
/// 完整 OAuth 授权按目标数据 ID 替换凭据与上游账号身份。
|
||||
pub(crate) fn prepare_reauthorization(
|
||||
/// 构建一次成功 RT exchange 的 CAS 写入,保留已存账号身份资料。
|
||||
///
|
||||
/// Refresh endpoint 已经完成本次 token 交换的授权;这里不对新 access
|
||||
/// token 重新执行身份验证。
|
||||
pub(crate) fn prepare_refreshed_oauth_rotation(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
current: LoadedCredential,
|
||||
secret: CodexOAuthSecret,
|
||||
access_token_expires_at: Option<DateTime<Utc>>,
|
||||
next_refresh_at: Option<DateTime<Utc>>,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
self.prepare_oauth_rotation(input, true)
|
||||
if current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| current.account.authentication_kind() != CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
{
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let mut data = CodexCredentialCodec::decode_complete(¤t.credential)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
oauth.access_token = secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(|value| value.expose_secret().to_owned());
|
||||
oauth.id_token = secret
|
||||
.id_token
|
||||
.as_ref()
|
||||
.map(|value| value.expose_secret().to_owned());
|
||||
let credential = CodexCredentialCodec::encode_complete(data)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let profile = ProviderAccountUpdate {
|
||||
account_id: current.account.id().clone(),
|
||||
name: current.account.name().to_owned(),
|
||||
email: current.account.email().map(str::to_owned),
|
||||
plan_type: current.account.plan_type().map(str::to_owned),
|
||||
};
|
||||
let credential = CredentialCasUpdate::new(
|
||||
current.account.id().clone(),
|
||||
current.account.revision(),
|
||||
profile.clone(),
|
||||
credential,
|
||||
secret.refresh_token.is_some(),
|
||||
optional_time(access_token_expires_at),
|
||||
optional_time(next_refresh_at),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
Ok(PreparedCodexCredentialRotation {
|
||||
profile,
|
||||
credential,
|
||||
replacement_identity: None,
|
||||
refresh_guards: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn prepare_oauth_rotation(
|
||||
@@ -512,24 +637,16 @@ impl CodexCredentialAdmin {
|
||||
input: RotateManagedCodexCredential,
|
||||
replace_identity: bool,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
let access_token_expires_at =
|
||||
required_time(input.verified_account.access_token_expires_at)?;
|
||||
let access_token_expires_at = input.verified_account.access_token_expires_at;
|
||||
let mut data = CodexCredentialCodec::decode_complete(&input.current.credential)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CodexCredentialAdminError::IdentityMismatch)?;
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
if input.current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| input.current.account.authentication_kind() != CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
|| (!replace_identity
|
||||
&& (input.current.account.upstream_account_id()
|
||||
!= Some(input.verified_account.chatgpt_account_id.as_str())
|
||||
|| input.current.account.upstream_user_id()
|
||||
!= input.verified_account.chatgpt_user_id
|
||||
|| oauth.principal.oauth_subject != input.verified_account.oauth_subject
|
||||
|| oauth.principal.poid != input.verified_account.poid))
|
||||
{
|
||||
return Err(CodexCredentialAdminError::IdentityMismatch);
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let replacement_identity = replace_identity.then(|| {
|
||||
ProviderAccountIdentity::new(
|
||||
@@ -538,14 +655,10 @@ impl CodexCredentialAdmin {
|
||||
)
|
||||
});
|
||||
if replace_identity {
|
||||
oauth
|
||||
.principal
|
||||
.oauth_subject
|
||||
.clone_from(&input.verified_account.oauth_subject);
|
||||
oauth
|
||||
.principal
|
||||
.poid
|
||||
.clone_from(&input.verified_account.poid);
|
||||
oauth.principal = Some(CodexCredentialPrincipal {
|
||||
oauth_subject: input.verified_account.oauth_subject.clone(),
|
||||
poid: input.verified_account.poid.clone(),
|
||||
});
|
||||
}
|
||||
oauth.access_token = input.secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = input
|
||||
@@ -572,7 +685,7 @@ impl CodexCredentialAdmin {
|
||||
profile.clone(),
|
||||
credential,
|
||||
input.secret.refresh_token.is_some(),
|
||||
Some(access_token_expires_at),
|
||||
optional_time(access_token_expires_at),
|
||||
optional_time(input.next_refresh_at),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
@@ -588,7 +701,6 @@ impl CodexCredentialAdmin {
|
||||
/// 有状态的 Codex 手工刷新边界;消费调用方刚读取的当前 credential 并准备 CAS。
|
||||
pub struct CodexCredentialAdminService {
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
}
|
||||
@@ -598,7 +710,6 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
formatter
|
||||
.debug_struct("CodexCredentialAdminService")
|
||||
.field("refresher", &"TokenRefresher")
|
||||
.field("verifier", &"CodexAccountIdentityVerifier")
|
||||
.field("leases", &"ProviderLeasePort")
|
||||
.field("runtime_policy", &"ProviderRuntimePolicyPort")
|
||||
.finish()
|
||||
@@ -608,19 +719,17 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
impl CodexCredentialAdminService {
|
||||
pub fn new(
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
refresher,
|
||||
verifier,
|
||||
leases,
|
||||
runtime_policy,
|
||||
}
|
||||
}
|
||||
|
||||
/// 官方 RT exchange + AT 身份验证;结果由 App 在同一 revision/audit 事务中提交。
|
||||
/// 官方 RT exchange;结果由 App 在同一 revision/audit 事务中提交。
|
||||
pub async fn manual_refresh(
|
||||
&self,
|
||||
current: LoadedCredential,
|
||||
@@ -632,7 +741,6 @@ impl CodexCredentialAdminService {
|
||||
}
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let expectation = current_identity_expectation(¤t.account, &runtime)?;
|
||||
let oauth = runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
@@ -677,9 +785,6 @@ impl CodexCredentialAdminService {
|
||||
.refresh(refresh_token.expose_secret())
|
||||
.await
|
||||
.map_err(map_refresh_failure)?;
|
||||
if tokens.access_token.trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
refresh_token: tokens
|
||||
@@ -688,24 +793,24 @@ impl CodexCredentialAdminService {
|
||||
.or_else(|| oauth.refresh_token.clone()),
|
||||
id_token: oauth.id_token.clone(),
|
||||
};
|
||||
let verified_account = self
|
||||
.verifier
|
||||
.verify(&secret, &expectation)
|
||||
.await
|
||||
.and_then(super::identity::CodexIdentityVerification::into_complete)
|
||||
.map_err(map_identity_error)?;
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
&account_id,
|
||||
verified_account.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)?;
|
||||
let mut prepared = CodexCredentialAdmin.prepare_rotation(RotateManagedCodexCredential {
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ManualRefresh,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
let access_token_expires_at = tokens
|
||||
.expires_in
|
||||
.and_then(|expires_in| SystemTime::now().checked_add(expires_in))
|
||||
.map(DateTime::<Utc>::from);
|
||||
// 正常预刷新由 worker 读取当时的 runtime policy 动态判断;这里仅清除
|
||||
// 之前瞬态失败留下的 retry-not-before。
|
||||
let next_refresh_at = None;
|
||||
let mut prepared = CodexCredentialAdmin.prepare_refreshed_oauth_rotation(
|
||||
current,
|
||||
secret,
|
||||
verified_account,
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
})?;
|
||||
)?;
|
||||
prepared.refresh_guards = Some(ProviderRefreshGuards {
|
||||
_capacity: capacity_guard,
|
||||
_account: account_guard,
|
||||
@@ -713,7 +818,10 @@ impl CodexCredentialAdminService {
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
/// 从 OAuth token 对补全账号事实,并归一到唯一 `NewProviderAccount` 写入路径。
|
||||
/// 归一导入 OAuth/agent-identity 凭据到唯一 `NewProviderAccount` 写入路径。
|
||||
///
|
||||
/// OAuth 导入只要取得 access token(直接提供或 RT exchange)即创建 Unknown
|
||||
/// 账号;资格资料由后续 worker 异步补齐。
|
||||
pub async fn prepare_import_document(
|
||||
&self,
|
||||
payload: Value,
|
||||
@@ -732,56 +840,54 @@ impl CodexCredentialAdminService {
|
||||
let mut account_ids = BTreeSet::new();
|
||||
let mut upstream_identities = BTreeSet::new();
|
||||
let mut accounts = Vec::with_capacity(candidates.len());
|
||||
let mut policy = None;
|
||||
for candidate in candidates {
|
||||
let prepared = match &candidate.authentication {
|
||||
ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token,
|
||||
} => {
|
||||
let account_id = format!("acct_{}", uuid::Uuid::now_v7().simple());
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (secret, verified_account) = self
|
||||
.verify_import_tokens(access_token.clone(), refresh_token.clone())
|
||||
let typed_account_id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?;
|
||||
let (secret, access_token_expires_at) = self
|
||||
.resolve_import_tokens(
|
||||
&typed_account_id,
|
||||
access_token.clone(),
|
||||
refresh_token.clone(),
|
||||
id_token.clone(),
|
||||
)
|
||||
.await?;
|
||||
if !upstream_identities.insert((
|
||||
verified_account.chatgpt_user_id.clone(),
|
||||
verified_account.chatgpt_account_id.clone(),
|
||||
)) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let refresh_policy = if let Some(policy) = policy {
|
||||
policy
|
||||
} else {
|
||||
let loaded = self
|
||||
.runtime_policy
|
||||
.load_refresh_policy()
|
||||
.await
|
||||
.map_err(|_| CodexCredentialAdminError::RefreshUnavailable)?;
|
||||
policy = Some(loaded);
|
||||
loaded
|
||||
};
|
||||
let next_refresh_at = refresh_time(
|
||||
refresh_policy,
|
||||
&ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?,
|
||||
verified_account.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)?;
|
||||
CodexCredentialAdmin.prepare_import(ImportCodexOAuthCredential {
|
||||
account_id,
|
||||
name: verified_account
|
||||
.email
|
||||
.clone()
|
||||
.filter(|name| !name.trim().is_empty())
|
||||
.unwrap_or_else(|| "Codex OAuth".to_owned()),
|
||||
secret,
|
||||
verified_account,
|
||||
next_refresh_at,
|
||||
enabled: true,
|
||||
})?
|
||||
// 导入材料按管理员输入原样保存;ID token 仅尽力补展示字段,
|
||||
// 解析失败不阻塞 AT/RT 入库。
|
||||
let mut metadata = secret
|
||||
.id_token
|
||||
.as_ref()
|
||||
.and_then(|token| {
|
||||
super::types::parse_id_token_metadata(token.expose_secret()).ok()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
apply_import_metadata(&mut metadata, &candidate);
|
||||
CodexCredentialAdmin.prepare_unresolved_oauth(
|
||||
UnresolvedCodexOAuthCredential {
|
||||
account_id,
|
||||
name: candidate
|
||||
.name
|
||||
.clone()
|
||||
.or_else(|| candidate.email.clone())
|
||||
.or_else(|| metadata.email.clone())
|
||||
.filter(|name| !name.trim().is_empty())
|
||||
.unwrap_or_else(|| "Codex OAuth".to_owned()),
|
||||
secret,
|
||||
metadata,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
},
|
||||
)?
|
||||
}
|
||||
ParsedCodexAuthentication::AgentIdentity { .. } => {
|
||||
let account_id = candidate
|
||||
@@ -796,20 +902,20 @@ impl CodexCredentialAdminService {
|
||||
import_runtime_state(candidate.status.as_deref())?;
|
||||
let mut account =
|
||||
self.prepare_agent_identity_import(&candidate, account_id, enabled)?;
|
||||
if !upstream_identities.insert((
|
||||
account.account.upstream_user_id().to_owned(),
|
||||
account
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?
|
||||
.to_owned(),
|
||||
)) {
|
||||
let upstream_user_id = account
|
||||
.account
|
||||
.upstream_user_id()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?
|
||||
.to_owned();
|
||||
let upstream_account_id = account
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?
|
||||
.to_owned();
|
||||
if !upstream_identities.insert((upstream_user_id, upstream_account_id)) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
account.account =
|
||||
account
|
||||
.account
|
||||
.with_runtime_state(enabled, availability, None);
|
||||
account.account = account.account.with_runtime_state(enabled, availability);
|
||||
account
|
||||
}
|
||||
};
|
||||
@@ -818,35 +924,27 @@ impl CodexCredentialAdminService {
|
||||
Ok(PreparedCodexAccountImport { accounts })
|
||||
}
|
||||
|
||||
async fn verify_import_tokens(
|
||||
async fn resolve_import_tokens(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
) -> Result<(CodexOAuthSecret, CodexAccountProfile), CodexCredentialAdminError> {
|
||||
let access_token = access_token
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
.filter(|token| !token.is_empty());
|
||||
let refresh_token = refresh_token
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
.filter(|token| !token.is_empty());
|
||||
id_token: Option<String>,
|
||||
) -> Result<(CodexOAuthSecret, Option<DateTime<Utc>>), CodexCredentialAdminError> {
|
||||
let id_token = id_token.map(SecretString::from);
|
||||
if let Some(access_token) = access_token {
|
||||
let access_token_expires_at = parse_access_token_expiration(&access_token);
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(access_token),
|
||||
refresh_token: refresh_token.clone().map(SecretString::from),
|
||||
id_token: None,
|
||||
refresh_token: refresh_token.map(SecretString::from),
|
||||
id_token,
|
||||
};
|
||||
match self.verifier.verify_import(&secret).await {
|
||||
Ok(profile) => return Ok((secret, profile)),
|
||||
Err(CodexIdentityVerificationError::Unavailable) => {
|
||||
return Err(CodexCredentialAdminError::IdentityUnavailable);
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Rejected) if refresh_token.is_none() => {
|
||||
return Err(CodexCredentialAdminError::IdentityRejected);
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Rejected) => {}
|
||||
}
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ImportDirect,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
return Ok((secret, access_token_expires_at));
|
||||
}
|
||||
let refresh_token = refresh_token.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
let tokens = self
|
||||
@@ -854,9 +952,6 @@ impl CodexCredentialAdminService {
|
||||
.refresh(&refresh_token)
|
||||
.await
|
||||
.map_err(map_refresh_failure)?;
|
||||
if tokens.access_token.trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
// RT 未轮换时仍保留导入提供的 RT,保证本次补全不会丢失后续刷新能力。
|
||||
@@ -864,14 +959,34 @@ impl CodexCredentialAdminService {
|
||||
.refresh_token
|
||||
.map(SecretString::from)
|
||||
.or_else(|| Some(SecretString::from(refresh_token))),
|
||||
id_token: None,
|
||||
id_token,
|
||||
};
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_import(&secret)
|
||||
.await
|
||||
.map_err(map_identity_error)?;
|
||||
Ok((secret, profile))
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ImportRefreshToken,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
let access_token_expires_at = tokens
|
||||
.expires_in
|
||||
.and_then(|expires_in| SystemTime::now().checked_add(expires_in))
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok((secret, access_token_expires_at))
|
||||
}
|
||||
|
||||
fn record_recovery_log(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
account_id: Option<&str>,
|
||||
secret: &CodexOAuthSecret,
|
||||
) {
|
||||
record_oauth_recovery(
|
||||
operation,
|
||||
account_id,
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
}
|
||||
|
||||
fn prepare_agent_identity_import(
|
||||
@@ -923,7 +1038,7 @@ impl CodexCredentialAdminService {
|
||||
account_id,
|
||||
provider,
|
||||
name,
|
||||
upstream_user_id,
|
||||
Some(upstream_user_id),
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY.to_owned(),
|
||||
revision,
|
||||
None,
|
||||
@@ -933,7 +1048,7 @@ impl CodexCredentialAdminService {
|
||||
Some(upstream_account_id),
|
||||
candidate.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(enabled, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(enabled, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(false, None);
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
@@ -942,56 +1057,10 @@ impl CodexCredentialAdminService {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_identity_expectation(
|
||||
account: &ProviderAccount,
|
||||
credential: &super::security::CodexRuntimeCredential,
|
||||
) -> Result<CodexIdentityExpectation, CodexCredentialAdminError> {
|
||||
let account_id = account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
let principal = credential
|
||||
.principal
|
||||
.as_ref()
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
CodexIdentityExpectation::current(
|
||||
principal.oauth_subject.clone(),
|
||||
principal.poid.clone(),
|
||||
account_id.to_owned(),
|
||||
account.upstream_user_id().to_owned(),
|
||||
credential.installation_id.clone(),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)
|
||||
}
|
||||
|
||||
fn required_time(
|
||||
value: Option<chrono::DateTime<chrono::Utc>>,
|
||||
) -> Result<SystemTime, CodexCredentialAdminError> {
|
||||
value
|
||||
.map(SystemTime::from)
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)
|
||||
}
|
||||
|
||||
fn optional_time(value: Option<chrono::DateTime<chrono::Utc>>) -> Option<SystemTime> {
|
||||
value.map(SystemTime::from)
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_time(
|
||||
policy: ProviderRefreshPolicy,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: Option<DateTime<Utc>>,
|
||||
has_refresh_token: bool,
|
||||
) -> Result<Option<DateTime<Utc>>, CodexCredentialAdminError> {
|
||||
if !has_refresh_token {
|
||||
return Ok(None);
|
||||
}
|
||||
let expires_at = required_time(access_token_expires_at)?;
|
||||
policy
|
||||
.next_attempt_at(account_id, expires_at, SystemTime::now())
|
||||
.map(DateTime::<Utc>::from)
|
||||
.map(Some)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)
|
||||
}
|
||||
|
||||
fn cpr_status(account: &ProviderAccount) -> &'static str {
|
||||
if !account.enabled() {
|
||||
return "disabled";
|
||||
@@ -1000,9 +1069,7 @@ fn cpr_status(account: &ProviderAccount) -> &'static str {
|
||||
AccountAvailability::QuotaExhausted => "quota_exhausted",
|
||||
AccountAvailability::Expired | AccountAvailability::Invalid => "expired",
|
||||
AccountAvailability::Banned => "banned",
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Cooldown => "active",
|
||||
AccountAvailability::Unknown | AccountAvailability::Ready => "active",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1021,15 +1088,6 @@ const fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError
|
||||
}
|
||||
}
|
||||
|
||||
const fn map_identity_error(error: CodexIdentityVerificationError) -> CodexCredentialAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => CodexCredentialAdminError::IdentityRejected,
|
||||
CodexIdentityVerificationError::Unavailable => {
|
||||
CodexCredentialAdminError::IdentityUnavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_import_document(
|
||||
payload: &Value,
|
||||
) -> Result<Vec<ParsedCodexImportAccount>, CodexCredentialAdminError> {
|
||||
@@ -1042,20 +1100,28 @@ fn parse_import_document(
|
||||
if !is_openai_oauth_candidate(value) {
|
||||
continue;
|
||||
}
|
||||
accounts.push(ParsedCodexImportAccount {
|
||||
id: None,
|
||||
name: None,
|
||||
email: None,
|
||||
plan_type: None,
|
||||
chatgpt_account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
authentication: parse_oauth_import_tokens(value)?,
|
||||
status: None,
|
||||
});
|
||||
accounts.push(parse_oauth_import_account(value)?);
|
||||
}
|
||||
Ok(accounts)
|
||||
}
|
||||
|
||||
fn parse_oauth_import_account(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
let credentials = value.get("credentials").unwrap_or(value);
|
||||
Ok(ParsedCodexImportAccount {
|
||||
id: first_string(value, &["id"]),
|
||||
name: first_string(value, &["name", "label"]),
|
||||
email: first_string(value, &["email"]).or_else(|| first_string(credentials, &["email"])),
|
||||
plan_type: first_string(value, &["plan_type", "planType"])
|
||||
.or_else(|| first_string(credentials, &["plan_type", "planType"])),
|
||||
chatgpt_account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
authentication: parse_oauth_import_tokens(value)?,
|
||||
status: first_string(value, &["status"]),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_agent_identity_account(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
@@ -1131,6 +1197,7 @@ fn parse_oauth_import_tokens(
|
||||
) -> Result<ParsedCodexAuthentication, CodexCredentialAdminError> {
|
||||
let mut access_token = None;
|
||||
let mut refresh_token = None;
|
||||
let mut id_token = None;
|
||||
let mut pending = vec![value];
|
||||
while let Some(current) = pending.pop() {
|
||||
match current {
|
||||
@@ -1139,18 +1206,14 @@ fn parse_oauth_import_tokens(
|
||||
let token = match key.as_str() {
|
||||
"access_token" | "accessToken" | "at" | "token" => &mut access_token,
|
||||
"refresh_token" | "refreshToken" | "rt" => &mut refresh_token,
|
||||
"id_token" | "idToken" => &mut id_token,
|
||||
_ => {
|
||||
pending.push(value);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let value = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
if token.replace(value.to_owned()).is_some() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
if let Some(value) = value.as_str() {
|
||||
*token = Some(value.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1164,6 +1227,7 @@ fn parse_oauth_import_tokens(
|
||||
Ok(ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1207,14 +1271,25 @@ fn first_string(value: &Value, keys: &[&str]) -> Option<String> {
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn normalize_bearer(value: &str) -> String {
|
||||
fn apply_import_metadata(metadata: &mut CodexOAuthMetadata, candidate: &ParsedCodexImportAccount) {
|
||||
if metadata.email.is_none() {
|
||||
metadata.email = candidate
|
||||
.email
|
||||
.as_deref()
|
||||
.and_then(|value| normalize_import_text(value, MAX_IMPORT_EMAIL_BYTES));
|
||||
}
|
||||
if metadata.chatgpt_plan_type.is_none() {
|
||||
metadata.chatgpt_plan_type = candidate
|
||||
.plan_type
|
||||
.as_deref()
|
||||
.and_then(|value| normalize_import_text(value, MAX_IMPORT_PLAN_BYTES));
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_import_text(value: &str, max_bytes: usize) -> Option<String> {
|
||||
let value = value.trim();
|
||||
value
|
||||
.strip_prefix("Bearer ")
|
||||
.or_else(|| value.strip_prefix("bearer "))
|
||||
.unwrap_or(value)
|
||||
.trim()
|
||||
.to_owned()
|
||||
(!value.is_empty() && value.len() <= max_bytes && !value.chars().any(char::is_control))
|
||||
.then(|| value.to_owned())
|
||||
}
|
||||
|
||||
fn import_runtime_state(
|
||||
|
||||
@@ -10,19 +10,33 @@ use crate::transport::request::derive_conversation_anchor;
|
||||
pub(crate) fn derive_codex_session_affinity_key(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<ProviderSessionAffinityKey> {
|
||||
if let Some(conversation_id) = request
|
||||
let session_key = if let Some(conversation_id) = request
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
if let Ok(key) = ProviderSessionAffinityKey::try_new(conversation_id.to_owned()) {
|
||||
return Some(key);
|
||||
Some(key)
|
||||
} else {
|
||||
opaque_affinity_key("local-conversation", conversation_id)
|
||||
}
|
||||
return opaque_affinity_key("local-conversation", conversation_id);
|
||||
}
|
||||
let (domain, value) = derive_conversation_anchor(request)?;
|
||||
opaque_affinity_key(domain, &value)
|
||||
} else {
|
||||
let (domain, value) = derive_conversation_anchor(request)?;
|
||||
opaque_affinity_key(domain, &value)
|
||||
};
|
||||
|
||||
let session_discriminator = request.subagent_kind();
|
||||
session_key.and_then(|session_key| match session_discriminator.as_deref() {
|
||||
None => Some(session_key),
|
||||
Some(discriminator) => {
|
||||
// 仅扩展原会话锚点的键空间;后续查找、绑定和调度仍使用同一套亲和逻辑。
|
||||
opaque_affinity_key(
|
||||
"subagent-session",
|
||||
&format!("{discriminator}\0{}", session_key.expose_to_store()),
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn opaque_affinity_key(domain: &str, value: &str) -> Option<ProviderSessionAffinityKey> {
|
||||
|
||||
@@ -783,7 +783,6 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool
|
||||
&& match account.availability() {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::QuotaExhausted => true,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,9 +5,9 @@ mod affinity;
|
||||
mod agent_identity;
|
||||
mod catalog;
|
||||
mod cookie;
|
||||
mod identity;
|
||||
mod oauth;
|
||||
mod quota;
|
||||
mod recovery_log;
|
||||
mod refresh;
|
||||
mod repository;
|
||||
mod security;
|
||||
@@ -15,7 +15,6 @@ mod selector;
|
||||
pub mod token_client;
|
||||
mod types;
|
||||
|
||||
pub(crate) use admin::refresh_time;
|
||||
pub(crate) use affinity::{
|
||||
derive_codex_cyber_policy_session_key, derive_codex_session_affinity_key,
|
||||
};
|
||||
@@ -38,13 +37,6 @@ pub use catalog::{
|
||||
CodexCredentialCatalogSnapshot, CodexPlanCatalog,
|
||||
};
|
||||
pub use cookie::{CodexCookiePolicy, CookiePolicyError};
|
||||
pub use identity::{
|
||||
CodexAccountIdentityService, CodexAccountIdentityVerifier, CodexAuthenticatedAccount,
|
||||
CodexAuthenticatedAccountSource, CodexIdentityExpectation, CodexIdentityVerification,
|
||||
CodexIdentityVerificationError, CodexJwksSource, CodexJwtIdentityVerifier, CodexSignedIdentity,
|
||||
CodexSignedIdentityVerifier, OFFICIAL_OPENAI_API_AUDIENCE, OFFICIAL_OPENAI_ISSUER,
|
||||
OFFICIAL_OPENAI_JWKS_URI, ReqwestCodexAuthenticatedAccountSource, ReqwestOpenAiJwksSource,
|
||||
};
|
||||
pub use oauth::{
|
||||
CodexOAuthAdmin, CodexOAuthAdminError, CodexOAuthAdminService, CodexOAuthAuthorizationStarted,
|
||||
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
//! Codex Authorization Code + PKCE/OIDC 管理流。
|
||||
|
||||
use std::{fmt, sync::Arc, time::Duration};
|
||||
use std::{
|
||||
fmt,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
@@ -13,9 +17,8 @@ use gateway_admin::model::{
|
||||
},
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
LoadedCredential, NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderRuntimePolicyPort;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
@@ -23,16 +26,16 @@ use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::admin::{
|
||||
CodexCredentialAdmin, CodexCredentialAdminError, ImportCodexOAuthCredential,
|
||||
PreparedCodexCredentialRotation, RotateManagedCodexCredential, refresh_time,
|
||||
};
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
CodexCredentialAdmin, CodexCredentialAdminError, PreparedCodexCredentialRotation,
|
||||
UnresolvedCodexOAuthCredential,
|
||||
};
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{
|
||||
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
|
||||
OFFICIAL_CODEX_OAUTH_CLIENT_ID, OFFICIAL_CODEX_REDIRECT_URI,
|
||||
};
|
||||
use super::types::parse_id_token_metadata;
|
||||
|
||||
const AUTHORIZATION_ENDPOINT: &str = "https://auth.openai.com/oauth/authorize";
|
||||
const AUTHORIZATION_SCOPE: &str = "openid profile email offline_access";
|
||||
@@ -316,8 +319,6 @@ pub enum CodexOAuthAdminError {
|
||||
CallbackRejected,
|
||||
#[error("Codex OAuth token exchange was rejected")]
|
||||
TokenRejected,
|
||||
#[error("Codex OAuth identity verification was rejected")]
|
||||
IdentityRejected,
|
||||
#[error("Codex OAuth upstream is unavailable")]
|
||||
UpstreamUnavailable,
|
||||
#[error("Codex OAuth exchange send state is ambiguous")]
|
||||
@@ -344,9 +345,7 @@ pub trait CodexOAuthAdmin: Send + Sync {
|
||||
pub struct CodexOAuthAdminService {
|
||||
pending: Arc<dyn CodexOAuthPendingStore>,
|
||||
exchanger: Arc<dyn AuthorizationCodeExchanger>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
store: Arc<dyn ProviderAccountStore>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
credentials: CodexCredentialAdmin,
|
||||
oauth_client_id: String,
|
||||
}
|
||||
@@ -396,17 +395,13 @@ impl CodexOAuthAdminService {
|
||||
pub fn new(
|
||||
pending: Arc<dyn CodexOAuthPendingStore>,
|
||||
exchanger: Arc<dyn AuthorizationCodeExchanger>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
store: Arc<dyn ProviderAccountStore>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
credentials: CodexCredentialAdmin,
|
||||
) -> Self {
|
||||
Self {
|
||||
pending,
|
||||
exchanger,
|
||||
verifier,
|
||||
store,
|
||||
runtime_policy,
|
||||
credentials,
|
||||
oauth_client_id: OFFICIAL_CODEX_OAUTH_CLIENT_ID.to_owned(),
|
||||
}
|
||||
@@ -487,8 +482,6 @@ impl CodexOAuthAdminService {
|
||||
callback_url: &str,
|
||||
) -> Result<(PendingAuthorizationMutation, CompletedCodexOAuthCredential), CodexOAuthAdminError>
|
||||
{
|
||||
let (mut secret, id_token) = self.exchange_pending(&pending, callback_url).await?;
|
||||
let mutation = pending.mutation.clone();
|
||||
let current = if let Some(target) = pending.reauthorization() {
|
||||
let current = self
|
||||
.store
|
||||
@@ -499,56 +492,43 @@ impl CodexOAuthAdminService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let expectation = CodexIdentityExpectation::default();
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_authorization(&secret, &id_token, &pending.nonce, &expectation)
|
||||
.await
|
||||
.and_then(super::identity::CodexIdentityVerification::into_complete)
|
||||
.map_err(map_identity_error)?;
|
||||
let fallback_refresh_token = current
|
||||
.as_ref()
|
||||
.map(current_oauth_refresh_token)
|
||||
.transpose()?
|
||||
.flatten();
|
||||
let (mut secret, id_token, expires_in) = self
|
||||
.exchange_pending(&pending, callback_url, fallback_refresh_token)
|
||||
.await?;
|
||||
let mutation = pending.mutation.clone();
|
||||
let access_token_expires_at = expires_in
|
||||
.and_then(|expires_in| SystemTime::now().checked_add(expires_in))
|
||||
.map(DateTime::<Utc>::from);
|
||||
let metadata = parse_id_token_metadata(id_token.expose_secret())
|
||||
.map_err(|_| CodexOAuthAdminError::TokenRejected)?;
|
||||
secret.id_token = Some(id_token);
|
||||
let policy = self
|
||||
.runtime_policy
|
||||
.load_refresh_policy()
|
||||
.await
|
||||
.map_err(|_| CodexOAuthAdminError::StorageUnavailable)?;
|
||||
let credential = if let Some(current) = current {
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
current.account.id(),
|
||||
profile.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)
|
||||
.map_err(map_admin_error)?;
|
||||
CompletedCodexOAuthCredential::Reauthorize(
|
||||
self.credentials
|
||||
.prepare_reauthorization(RotateManagedCodexCredential {
|
||||
.prepare_refreshed_oauth_rotation(
|
||||
current,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
next_refresh_at,
|
||||
})
|
||||
access_token_expires_at,
|
||||
None,
|
||||
)
|
||||
.map_err(map_admin_error)?,
|
||||
)
|
||||
} else {
|
||||
let account_id = format!("acct_{}", Uuid::now_v7().simple());
|
||||
let typed_account_id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
&typed_account_id,
|
||||
profile.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)
|
||||
.map_err(map_admin_error)?;
|
||||
CompletedCodexOAuthCredential::Create(
|
||||
self.credentials
|
||||
.prepare_import(ImportCodexOAuthCredential {
|
||||
.prepare_unresolved_oauth(UnresolvedCodexOAuthCredential {
|
||||
account_id,
|
||||
name: pending.name,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
next_refresh_at,
|
||||
metadata,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
})
|
||||
.map_err(map_admin_error)?,
|
||||
@@ -654,7 +634,15 @@ impl CodexOAuthAdminService {
|
||||
&self,
|
||||
pending: &CodexPendingAuthorization,
|
||||
callback_url: &str,
|
||||
) -> Result<(super::types::CodexOAuthSecret, SecretString), CodexOAuthAdminError> {
|
||||
fallback_refresh_token: Option<SecretString>,
|
||||
) -> Result<
|
||||
(
|
||||
super::types::CodexOAuthSecret,
|
||||
SecretString,
|
||||
Option<Duration>,
|
||||
),
|
||||
CodexOAuthAdminError,
|
||||
> {
|
||||
let (code, callback_state) = callback_parts(callback_url)?;
|
||||
if !constant_time_equal(
|
||||
pending.state.expose_secret().as_bytes(),
|
||||
@@ -670,10 +658,36 @@ impl CodexOAuthAdminService {
|
||||
})
|
||||
.await
|
||||
.map_err(map_exchange_error)?;
|
||||
Ok((tokens.secret, tokens.id_token))
|
||||
let mut secret = tokens.secret;
|
||||
if secret.refresh_token.is_none() {
|
||||
secret.refresh_token = fallback_refresh_token;
|
||||
}
|
||||
record_oauth_recovery(
|
||||
CodexOAuthRecoveryOperation::AuthorizationCode,
|
||||
pending
|
||||
.reauthorization()
|
||||
.map(|account_id| account_id.as_str()),
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
Ok((secret, tokens.id_token, tokens.expires_in))
|
||||
}
|
||||
}
|
||||
|
||||
fn current_oauth_refresh_token(
|
||||
current: &LoadedCredential,
|
||||
) -> Result<Option<SecretString>, CodexOAuthAdminError> {
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
Ok(runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.and_then(|oauth| oauth.refresh_token.clone()))
|
||||
}
|
||||
|
||||
fn authorization_url(
|
||||
pending: &CodexPendingAuthorization,
|
||||
oauth_client_id: &str,
|
||||
@@ -699,27 +713,19 @@ fn authorization_url(
|
||||
|
||||
fn callback_parts(value: &str) -> Result<(SecretString, SecretString), CodexOAuthAdminError> {
|
||||
let url = Url::parse(value).map_err(|_| CodexOAuthAdminError::CallbackRejected)?;
|
||||
if url.scheme() != "http"
|
||||
|| url.host_str() != Some("localhost")
|
||||
|| url.port() != Some(1455)
|
||||
|| url.path() != "/auth/callback"
|
||||
|| url.fragment().is_some()
|
||||
{
|
||||
return Err(CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
let mut code = None;
|
||||
let mut state = None;
|
||||
for (key, value) in url.query_pairs() {
|
||||
match key.as_ref() {
|
||||
"code" => set_unique_callback_parameter(&mut code, value.into_owned())?,
|
||||
"state" => set_unique_callback_parameter(&mut state, value.into_owned())?,
|
||||
// OAuth 服务可能附加 `scope`、`iss` 等标准参数。
|
||||
// 固定回调地址与唯一的 `code`/`state` 才是安全边界,无关参数不参与校验。
|
||||
// 回调地址只承载 query 参数;安全绑定由唯一的 code/state、
|
||||
// 服务端保存的 state 和 PKCE token exchange 共同完成。
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let code = code.filter(|value| valid_secret(value));
|
||||
let state = state.filter(|value| valid_secret(value));
|
||||
let code = code.filter(|value| !value.is_empty());
|
||||
let state = state.filter(|value| !value.is_empty());
|
||||
match (code, state) {
|
||||
(Some(code), Some(state)) => Ok((SecretString::from(code), SecretString::from(state))),
|
||||
_ => Err(CodexOAuthAdminError::CallbackRejected),
|
||||
@@ -813,13 +819,6 @@ fn map_exchange_error(error: AuthorizationCodeExchangeError) -> CodexOAuthAdminE
|
||||
}
|
||||
}
|
||||
|
||||
fn map_identity_error(error: CodexIdentityVerificationError) -> CodexOAuthAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => CodexOAuthAdminError::IdentityRejected,
|
||||
CodexIdentityVerificationError::Unavailable => CodexOAuthAdminError::UpstreamUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
fn map_admin_error(_: CodexCredentialAdminError) -> CodexOAuthAdminError {
|
||||
CodexOAuthAdminError::Credential
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,102 @@
|
||||
//! 权威额度快照的 availability 转换。
|
||||
//!
|
||||
//! 已触顶的 core primary 会立即把可调度账号标为 `QuotaExhausted`;
|
||||
//! `QuotaExhausted -> Ready` 则只能由同一 credential revision 的权威 usage 快照证明:
|
||||
//! 新快照未耗尽、core primary reset 已推进到下一窗口,且该窗口用量低于 10%。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::engine::credential::AccountAvailability;
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::snapshot::CodexAccountQuotaSnapshot;
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0;
|
||||
|
||||
/// 基于成功的权威 usage 快照收敛可调度账号的额度状态。
|
||||
///
|
||||
/// 只接受上游明确拒绝使用且 core primary 触顶的证据;secondary、附加桶、
|
||||
/// credits 与 spend control 都不能单独把整个账号标成额度耗尽。
|
||||
pub(crate) fn authoritative_quota_snapshot_state(
|
||||
current: AccountAvailability,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
match current {
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown
|
||||
if snapshot.authoritative_core_primary_exhausted() =>
|
||||
{
|
||||
Some(AccountAvailability::QuotaExhausted)
|
||||
}
|
||||
AccountAvailability::QuotaExhausted => {
|
||||
quota_success_state(current, snapshot, previous_reset_at)
|
||||
}
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown => None,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独覆盖
|
||||
/// 一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
|
||||
///
|
||||
/// `previous_reset_at` 是写入新快照前读取的 core primary 已确认耗尽窗口。上游可能只
|
||||
/// 调整 reset 时间而尚未重置额度,因此还要求同一新鲜 primary 窗口的用量低于 10%。
|
||||
pub(crate) fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
if current != AccountAvailability::QuotaExhausted || snapshot.fact().exhausted() {
|
||||
return None;
|
||||
}
|
||||
let previous_reset_at = previous_reset_at?;
|
||||
let primary_window = snapshot.core_primary_window()?;
|
||||
let refreshed_reset_at = primary_window.reset_at().map(SystemTime::from)?;
|
||||
let used_percent = primary_window.used_percent()?;
|
||||
(refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT)
|
||||
.then_some(AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
|
||||
let CodexClientError::Upstream { status, body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) {
|
||||
return Some(AccountAvailability::Banned);
|
||||
}
|
||||
match *status {
|
||||
// 402:真额度耗尽 → QuotaExhausted(worker 按 reset_at 自动恢复)。
|
||||
StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted),
|
||||
// usage 401/403 不能单独判定账号不可用:保持现有状态,交给 OAuth 或真实请求确认。
|
||||
// 429/503 同样不写 availability,仅记失败待重试。
|
||||
StatusCode::UNAUTHORIZED
|
||||
| StatusCode::FORBIDDEN
|
||||
| StatusCode::TOO_MANY_REQUESTS
|
||||
| StatusCode::SERVICE_UNAVAILABLE => None,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 配额接口将账号变为非 ready 状态时可安全持久化的上游原因码。
|
||||
pub(crate) fn quota_state_reason(error: &CodexClientError) -> Option<&'static str> {
|
||||
match quota_state_transition(error)? {
|
||||
AccountAvailability::Banned => Some("deactivated_workspace"),
|
||||
AccountAvailability::QuotaExhausted => Some("payment_required"),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(body: &str) -> bool {
|
||||
serde_json::from_str::<Value>(body)
|
||||
.ok()
|
||||
.is_some_and(|value| {
|
||||
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,734 @@
|
||||
//! 快照/窗口解析、聚合、O(1) 滚动与调度信号。
|
||||
//!
|
||||
//! Provider raw quota JSON 的唯一解析者:产出 [`CodexAccountQuotaSnapshot`]
|
||||
//! 供服务编排层消费;`limit_reached` 是快照级事实,不能只从子窗口反推。
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaObservation,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL};
|
||||
|
||||
const CORE_PRIMARY_WINDOW_KEY: &str = "core-primary";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CodexQuotaFact {
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
exhausted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CodexQuotaWindowKind {
|
||||
ShortTerm,
|
||||
Weekly,
|
||||
Monthly,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CodexQuotaWindowRole {
|
||||
Primary,
|
||||
Secondary,
|
||||
Monthly,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CodexQuotaWindow {
|
||||
key: String,
|
||||
source: String,
|
||||
account_wide: bool,
|
||||
/// 上游提供的限流桶名称(`metered_feature`/`limit_name`/`limitId`);用于非核心桶的展示前缀。
|
||||
limit_name: Option<String>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
role: CodexQuotaWindowRole,
|
||||
window_seconds: Option<u64>,
|
||||
used_percent: Option<f64>,
|
||||
reset_at: Option<DateTime<Utc>>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl CodexQuotaWindow {
|
||||
#[must_use]
|
||||
pub fn key(&self) -> &str {
|
||||
&self.key
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn source(&self) -> &str {
|
||||
&self.source
|
||||
}
|
||||
|
||||
/// 该窗口是否覆盖账号全部 Codex 请求,可使用账号级本地用量聚合。
|
||||
#[must_use]
|
||||
pub const fn is_account_wide(&self) -> bool {
|
||||
self.account_wide
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn limit_name(&self) -> Option<&str> {
|
||||
self.limit_name.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> CodexQuotaWindowKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn role(&self) -> CodexQuotaWindowRole {
|
||||
self.role
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn window_seconds(&self) -> Option<u64> {
|
||||
self.window_seconds
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn used_percent(&self) -> Option<f64> {
|
||||
self.used_percent
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reset_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.reset_at
|
||||
}
|
||||
|
||||
/// 该窗口是否已触顶(`limit_reached` 或 `used_percent >= 100`)。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(&self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CodexAccountQuotaSnapshot {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
fact: CodexQuotaFact,
|
||||
authoritative_core_primary_exhausted: bool,
|
||||
windows: Vec<CodexQuotaWindow>,
|
||||
}
|
||||
|
||||
impl CodexAccountQuotaSnapshot {
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn observed_at(&self) -> SystemTime {
|
||||
self.observed_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn fact(&self) -> CodexQuotaFact {
|
||||
self.fact
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn windows(&self) -> &[CodexQuotaWindow] {
|
||||
&self.windows
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn core_primary_window(&self) -> Option<&CodexQuotaWindow> {
|
||||
self.windows
|
||||
.iter()
|
||||
.find(|window| window.key() == CORE_PRIMARY_WINDOW_KEY)
|
||||
}
|
||||
|
||||
/// 上游成功 usage 明确拒绝请求,且 canonical core primary 已确认触顶。
|
||||
#[must_use]
|
||||
pub(crate) const fn authoritative_core_primary_exhausted(&self) -> bool {
|
||||
self.authoritative_core_primary_exhausted
|
||||
}
|
||||
|
||||
/// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、
|
||||
/// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。
|
||||
/// 只作用于展示/调度投影,不写回持久化 raw JSON。
|
||||
///
|
||||
/// O(1) 计算:`next_reset = reset + (floor((now - reset) / window) + 1) * window`,
|
||||
/// 不做逐周期线性循环。
|
||||
#[must_use]
|
||||
pub fn roll_expired_windows(mut self, now: SystemTime) -> Self {
|
||||
for window in &mut self.windows {
|
||||
let Some(reset_at) = window.reset_at else {
|
||||
continue;
|
||||
};
|
||||
let reset = SystemTime::from(reset_at);
|
||||
if reset > now {
|
||||
continue;
|
||||
}
|
||||
window.used_percent = Some(0.0);
|
||||
window.limit_reached = false;
|
||||
let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) else {
|
||||
// 没有合法窗口时长:清除已过期的 reset,等待下一次权威观测。
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
let elapsed = now
|
||||
.duration_since(reset)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let periods = elapsed / window_seconds + 1;
|
||||
let Some(after) =
|
||||
reset.checked_add(Duration::from_secs(periods.saturating_mul(window_seconds)))
|
||||
else {
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
window.reset_at = Some(DateTime::<Utc>::from(after));
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexQuotaFact {
|
||||
#[must_use]
|
||||
pub const fn remaining_percent(&self) -> Option<u8> {
|
||||
self.remaining_percent
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn resets_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.resets_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn exhausted(&self) -> bool {
|
||||
self.exhausted
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let mut aggregate = QuotaAggregate::default();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
aggregate.observe_rate_limit(limit.rate_limit)?;
|
||||
}
|
||||
// spend_control 只提供 exhaustion 信号(`reached`),不生成窗口、
|
||||
// 不参与剩余量聚合(`individual_limit` 等字段无窗口语义)。
|
||||
if let Some(spend_control) = object.get("spend_control") {
|
||||
aggregate.observe_exhaustion_object(spend_control, "reached")?;
|
||||
}
|
||||
if let Some(credits) = object.get("credits") {
|
||||
aggregate.observe_exhaustion_object(credits, "overage_limit_reached")?;
|
||||
}
|
||||
if !aggregate.recognized {
|
||||
return Err(CodexCredentialQuotaError::InvalidCredentialData);
|
||||
}
|
||||
Ok(CodexQuotaFact {
|
||||
remaining_percent: aggregate.remaining_percent,
|
||||
resets_at: aggregate.resets_at,
|
||||
exhausted: aggregate.exhausted,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn quota_snapshot_from_observation(
|
||||
observation: &QuotaObservation,
|
||||
) -> Option<CodexAccountQuotaSnapshot> {
|
||||
let observed_at = observation.observed_at?;
|
||||
let quota = observation.quota.as_ref()?;
|
||||
parse_account_quota_snapshot(
|
||||
observation.account_id.clone(),
|
||||
observation.expected_revision,
|
||||
observed_at,
|
||||
&Value::Object(quota.expose_to_provider().clone()),
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// 调度投影缓存 TTL:`min(常规 TTL, 最近 active reset - now)`。
|
||||
/// reset 到期时缓存自然失效,不固定多等一个常规 TTL 周期。
|
||||
pub(crate) fn quota_projection_ttl(snapshot: &CodexAccountQuotaSnapshot) -> Option<Duration> {
|
||||
let age = SystemTime::now()
|
||||
.duration_since(snapshot.observed_at())
|
||||
.unwrap_or(Duration::ZERO);
|
||||
let normal_ttl = QUOTA_SCHEDULING_TTL.checked_sub(age)?;
|
||||
let now = SystemTime::now();
|
||||
let reset_ttl = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter_map(|window| window.reset_at())
|
||||
.map(SystemTime::from)
|
||||
.filter(|reset| *reset > now)
|
||||
.map(|reset| reset.duration_since(now).unwrap_or(Duration::ZERO))
|
||||
.min();
|
||||
let ttl = match reset_ttl {
|
||||
Some(reset_ttl) if reset_ttl < normal_ttl => reset_ttl,
|
||||
_ => normal_ttl,
|
||||
};
|
||||
(!ttl.is_zero()).then_some(ttl)
|
||||
}
|
||||
|
||||
/// 从快照的滚动窗口派生中立调度事实(`limit_reached` = 任一窗口触顶)。
|
||||
pub(crate) fn scheduling_signals_from_snapshot(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let rolled = snapshot.clone().roll_expired_windows(SystemTime::now());
|
||||
quota_scheduling_signals(&rolled)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_scheduling_signals(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let now = SystemTime::now();
|
||||
let mut limit_reached = false;
|
||||
let mut remaining_rank: Option<u64> = None;
|
||||
let mut reset_at: Option<SystemTime> = None;
|
||||
for window in &snapshot.windows {
|
||||
limit_reached |= window.limit_reached();
|
||||
if let Some(used) = window
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
{
|
||||
let rank = ((100.0 - used) * 100.0).round() as u64;
|
||||
remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank)));
|
||||
}
|
||||
if let Some(reset) = window
|
||||
.reset_at()
|
||||
.map(SystemTime::from)
|
||||
.filter(|reset| *reset > now)
|
||||
{
|
||||
reset_at = Some(reset_at.map_or(reset, |current| current.min(reset)));
|
||||
}
|
||||
}
|
||||
// 快照级 limit_reached 是调度排除的充分条件:即使没有任何窗口带
|
||||
// percent/reset(如顶层限流标记),也必须产出信号;不能返回 None 丢失排除。
|
||||
if limit_reached || remaining_rank.is_some() || reset_at.is_some() {
|
||||
Some(AccountQuotaSignals::new(
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_account_quota_snapshot(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
usage: &Value,
|
||||
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
|
||||
let fact = parse_codex_quota_usage(usage)?;
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let authoritative_core_primary_exhausted = authoritative_core_primary_exhausted(object)?;
|
||||
let mut windows = Vec::new();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
parse_rate_limit_windows(
|
||||
&limit.key,
|
||||
&limit.source,
|
||||
limit.account_wide,
|
||||
limit.limit_name.as_deref(),
|
||||
limit.rate_limit,
|
||||
&mut windows,
|
||||
)?;
|
||||
}
|
||||
windows.sort_by(|left, right| {
|
||||
quota_source_order(left.source())
|
||||
.cmp("a_source_order(right.source()))
|
||||
.then_with(|| left.key.cmp(&right.key))
|
||||
});
|
||||
Ok(CodexAccountQuotaSnapshot {
|
||||
account_id,
|
||||
credential_revision,
|
||||
observed_at,
|
||||
fact,
|
||||
authoritative_core_primary_exhausted,
|
||||
windows,
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_source_order(source: &str) -> u8 {
|
||||
match source {
|
||||
"codex" => 0,
|
||||
"code_review" => 1,
|
||||
_ => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号状态只能由顶层 canonical Codex 额度桶的明确拒绝来改变。
|
||||
///
|
||||
/// `fact.exhausted()` 和窗口 `limit_reached` 都会聚合或继承 secondary、附加桶等事实,
|
||||
/// 因此不能单独用来将整个账号标为 `QuotaExhausted`。
|
||||
fn authoritative_core_primary_exhausted(
|
||||
usage: &Map<String, Value>,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
let Some(rate_limit) = usage.get("rate_limit") else {
|
||||
return Ok(false);
|
||||
};
|
||||
if rate_limit.is_null() {
|
||||
return Ok(false);
|
||||
}
|
||||
let rate_limit = rate_limit
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
if optional_bool(rate_limit, "allowed")? != Some(false) {
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(primary_window) = rate_limit.get("primary_window") else {
|
||||
return Ok(false);
|
||||
};
|
||||
if primary_window.is_null() {
|
||||
return Ok(false);
|
||||
}
|
||||
let primary_window = primary_window
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let primary_window_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(primary_window, "limit_reached")?.unwrap_or(false)
|
||||
|| primary_window
|
||||
.get("used_percent")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.map(|value| value >= 100.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(false);
|
||||
Ok(primary_window_reached)
|
||||
}
|
||||
|
||||
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
|
||||
struct CanonicalRateLimit<'a> {
|
||||
key: String,
|
||||
source: String,
|
||||
account_wide: bool,
|
||||
limit_name: Option<String>,
|
||||
rate_limit: &'a Value,
|
||||
}
|
||||
|
||||
fn canonical_rate_limits(
|
||||
object: &Map<String, Value>,
|
||||
) -> Result<Vec<CanonicalRateLimit<'_>>, CodexCredentialQuotaError> {
|
||||
let mut limits = Vec::new();
|
||||
let has_primary_codex_rate_limit = object
|
||||
.get("rate_limit")
|
||||
.is_some_and(|value| !value.is_null());
|
||||
if let Some(rate_limit) = object.get("rate_limit") {
|
||||
// 官方 Codex 将顶层 `rate_limit` 固定识别为 `codex`,而不把
|
||||
// `active_limit` 投影成另一个展示桶。后者只是当前活动套餐的元数据。
|
||||
limits.push(CanonicalRateLimit {
|
||||
key: "core".to_owned(),
|
||||
source: "codex".to_owned(),
|
||||
account_wide: true,
|
||||
limit_name: None,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
if let Some(rate_limit) = object
|
||||
.get("code_review_rate_limit")
|
||||
.filter(|value| !value.is_null())
|
||||
{
|
||||
limits.push(CanonicalRateLimit {
|
||||
key: "code-review".to_owned(),
|
||||
source: "code_review".to_owned(),
|
||||
account_wide: false,
|
||||
limit_name: Some("code_review".to_owned()),
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
let Some(additional) = object
|
||||
.get("additional_rate_limits")
|
||||
.filter(|value| !value.is_null())
|
||||
else {
|
||||
return Ok(limits);
|
||||
};
|
||||
for (index, value) in additional
|
||||
.as_array()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let item = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let (source, limit_name) = canonical_rate_limit_source(item, index);
|
||||
// 部分上游响应会在 `additional_rate_limits` 中重复默认的 `codex`
|
||||
// 桶。它没有独立的 limit id,不能在账号面板再次展示。
|
||||
if has_primary_codex_rate_limit && is_codex_limit_id(&source) {
|
||||
continue;
|
||||
}
|
||||
if let Some(rate_limit) = item.get("rate_limit") {
|
||||
// 非 Codex additional 桶保留为独立快照;额外用索引生成窗口键,
|
||||
// 避免同名 additional 产生重复前端 key。
|
||||
limits.push(CanonicalRateLimit {
|
||||
key: format!("additional-{index}-{source}"),
|
||||
account_wide: is_codex_limit_id(&source),
|
||||
source,
|
||||
limit_name,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(limits)
|
||||
}
|
||||
|
||||
fn is_codex_limit_id(value: &str) -> bool {
|
||||
value.trim().eq_ignore_ascii_case("codex")
|
||||
}
|
||||
|
||||
fn canonical_rate_limit_source(
|
||||
item: &Map<String, Value>,
|
||||
index: usize,
|
||||
) -> (String, Option<String>) {
|
||||
let source = item
|
||||
.get("metered_feature")
|
||||
.or_else(|| item.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("additional-{index}"));
|
||||
let limit_name = item
|
||||
.get("limit_name")
|
||||
.or_else(|| item.get("metered_feature"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_owned);
|
||||
(source, limit_name)
|
||||
}
|
||||
|
||||
fn parse_rate_limit_windows(
|
||||
key_source: &str,
|
||||
source: &str,
|
||||
account_wide: bool,
|
||||
limit_name: Option<&str>,
|
||||
value: &Value,
|
||||
output: &mut Vec<CodexQuotaWindow>,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 顶层限流标记是快照级事实:任一窗口都要继承所属
|
||||
// rate_limit 的 `limit_reached=true` / `allowed=false`。
|
||||
let top_level_reached = optional_bool(object, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
for (name, role) in [
|
||||
("primary_window", CodexQuotaWindowRole::Primary),
|
||||
("secondary_window", CodexQuotaWindowRole::Secondary),
|
||||
] {
|
||||
let Some(window) = object.get(name) else {
|
||||
continue;
|
||||
};
|
||||
if window.is_null() {
|
||||
continue;
|
||||
}
|
||||
let window = window
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let window_seconds = optional_positive_u64(window, "limit_window_seconds")?
|
||||
.or(optional_positive_u64(window, "window_seconds")?)
|
||||
.or(optional_positive_u64(window, "window_minutes")?
|
||||
.and_then(|minutes| minutes.checked_mul(60)));
|
||||
let used_percent = window
|
||||
.get("used_percent")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.map(|value| value.clamp(0.0, 100.0))
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let reset_at = window
|
||||
.get("reset_at")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_i64()
|
||||
.filter(|value| *value > 0)
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value, 0))
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let limit_reached = top_level_reached
|
||||
|| window
|
||||
.get("limit_reached")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
|| used_percent.is_some_and(|used| used >= 100.0);
|
||||
let kind = quota_window_kind(window_seconds);
|
||||
output.push(CodexQuotaWindow {
|
||||
key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)),
|
||||
source: source.to_owned(),
|
||||
account_wide,
|
||||
limit_name: limit_name.map(str::to_owned),
|
||||
kind,
|
||||
role,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn optional_positive_u64(
|
||||
object: &Map<String, Value>,
|
||||
key: &str,
|
||||
) -> Result<Option<u64>, CodexCredentialQuotaError> {
|
||||
object
|
||||
.get(key)
|
||||
.map(|value| {
|
||||
value
|
||||
.as_u64()
|
||||
.filter(|value| *value > 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
const fn quota_window_kind(seconds: Option<u64>) -> CodexQuotaWindowKind {
|
||||
match seconds {
|
||||
Some(value) if value >= 17_100 && value <= 18_900 => CodexQuotaWindowKind::ShortTerm,
|
||||
Some(value) if value >= 574_560 && value <= 635_040 => CodexQuotaWindowKind::Weekly,
|
||||
Some(value) if value >= 2_462_400 && value <= 2_721_600 => CodexQuotaWindowKind::Monthly,
|
||||
_ => CodexQuotaWindowKind::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_key(value: &str) -> String {
|
||||
let mut key = value
|
||||
.chars()
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() {
|
||||
character.to_ascii_lowercase()
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
while key.contains("--") {
|
||||
key = key.replace("--", "-");
|
||||
}
|
||||
let key = key.trim_matches('-');
|
||||
if key.is_empty() {
|
||||
"quota".to_owned()
|
||||
} else {
|
||||
key.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind) -> &'static str {
|
||||
match kind {
|
||||
CodexQuotaWindowKind::ShortTerm => "five-hour",
|
||||
CodexQuotaWindowKind::Weekly => "weekly",
|
||||
CodexQuotaWindowKind::Monthly => "monthly",
|
||||
CodexQuotaWindowKind::Other => match role {
|
||||
CodexQuotaWindowRole::Primary => "primary",
|
||||
CodexQuotaWindowRole::Secondary => "secondary",
|
||||
CodexQuotaWindowRole::Monthly => "monthly",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct QuotaAggregate {
|
||||
recognized: bool,
|
||||
exhausted: bool,
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl QuotaAggregate {
|
||||
fn observe_rate_limit(&mut self, value: &Value) -> Result<(), CodexCredentialQuotaError> {
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
|
||||
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
for key in ["primary_window", "secondary_window"] {
|
||||
if let Some(window) = object.get(key) {
|
||||
self.observe_window(window)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_window(&mut self, value: &Value) -> Result<(), CodexCredentialQuotaError> {
|
||||
if value.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
if let Some(used) = object.get("used_percent") {
|
||||
let used = used
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 子窗口触顶(used>=100)是快照级 exhaustion 事实。
|
||||
self.exhausted |= used >= 100.0;
|
||||
let remaining = (100.0 - used).clamp(0.0, 100.0).round() as u8;
|
||||
self.remaining_percent = Some(
|
||||
self.remaining_percent
|
||||
.map_or(remaining, |current| current.min(remaining)),
|
||||
);
|
||||
}
|
||||
if let Some(reset) = object.get("reset_at") {
|
||||
let seconds = reset
|
||||
.as_i64()
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let reset = DateTime::<Utc>::from_timestamp(seconds, 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.resets_at = Some(self.resets_at.map_or(reset, |current| current.min(reset)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_exhaustion_object(
|
||||
&mut self,
|
||||
value: &Value,
|
||||
key: &str,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
if value.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, key)?.unwrap_or(false);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_bool(
|
||||
object: &Map<String, Value>,
|
||||
key: &str,
|
||||
) -> Result<Option<bool>, CodexCredentialQuotaError> {
|
||||
object
|
||||
.get(key)
|
||||
.map(|value| {
|
||||
value
|
||||
.as_bool()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
//! 成功取得 OAuth token 后写入独立恢复日志的记录。
|
||||
//!
|
||||
//! 此日志会刻意保存原始 AT/RT,避免后续数据库写入或校验失败时,已成功
|
||||
//! 交换的账号无法恢复。记录独立落盘,但沿用普通结构化日志的滚动和清理策略。
|
||||
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
|
||||
/// 产生可恢复 token 组的 OAuth 操作。
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum CodexOAuthRecoveryOperation {
|
||||
/// 直接提供 AT/RT 的导入。
|
||||
ImportDirect,
|
||||
/// 仅提供 RT 的导入,先交换得到 AT。
|
||||
ImportRefreshToken,
|
||||
/// 管理员手动触发的 RT 交换。
|
||||
ManualRefresh,
|
||||
/// 调度器触发的 RT 交换。
|
||||
ScheduledRefresh,
|
||||
/// 授权码加 PKCE 的 token 交换。
|
||||
AuthorizationCode,
|
||||
}
|
||||
|
||||
impl CodexOAuthRecoveryOperation {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ImportDirect => "import_direct",
|
||||
Self::ImportRefreshToken => "import_refresh_token",
|
||||
Self::ManualRefresh => "manual_refresh",
|
||||
Self::ScheduledRefresh => "scheduled_refresh",
|
||||
Self::AuthorizationCode => "authorization_code",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 将已取得的 AT/RT 写入独立恢复结构化日志。
|
||||
///
|
||||
/// 调用方必须在任何可能拒绝响应的本地校验、画像补全或数据库写入之前调用。
|
||||
/// 该调用不执行额外 I/O,也不返回业务错误。Host 负责恢复日志文件的分割、清理和
|
||||
/// 输出;日志系统不可用时按其既有策略处理,不能阻断 OAuth 业务流程。
|
||||
pub(crate) fn record_oauth_recovery(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
account_id: Option<&str>,
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
) {
|
||||
let has_refresh_token = refresh_token.is_some();
|
||||
let refresh_token = refresh_token.unwrap_or_default();
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
event = "oauth_recovery",
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
operation = operation.as_str(),
|
||||
account_id = account_id.unwrap_or_default(),
|
||||
access_token,
|
||||
refresh_token,
|
||||
has_refresh_token,
|
||||
"OAuth recovery token record"
|
||||
);
|
||||
}
|
||||
@@ -11,38 +11,123 @@ use gateway_core::provider_ports::{
|
||||
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
ProviderRuntimePolicyPort, ProviderStoreError, provider_refresh_backoff_at,
|
||||
provider_refresh_retry_at,
|
||||
};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerification,
|
||||
CodexIdentityVerificationError,
|
||||
};
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use super::token_client::{RefreshFailure, TokenPair, TokenRefresher};
|
||||
use super::types::{CodexCredentialPrincipal, CodexOAuthSecret, RotateCodexCredential};
|
||||
use super::types::CodexOAuthSecret;
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const MAX_REFRESH_BATCH: u32 = 1_000;
|
||||
/// 连续失败计数窗口;每次瞬态失败刷新该 TTL,静默满窗后计数过期归零。
|
||||
const REFRESH_BACKOFF_WINDOW: Duration = Duration::from_secs(30 * 60);
|
||||
/// 窗口内先执行五次指数退避,再进入固定恢复周期。
|
||||
const REFRESH_BACKOFF_MAX_ATTEMPTS: u32 = 5;
|
||||
/// 耗尽指数退避后的 OAuth 恢复周期。
|
||||
const REFRESH_RECOVERY_DELAY: Duration = Duration::from_secs(10 * 60);
|
||||
/// 过期 AT 仍允许 RT 恢复的最长窗口。
|
||||
const REFRESH_RECOVERY_WINDOW: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
fn refresh_due_at(account: &ProviderAccount, now: SystemTime) -> Option<SystemTime> {
|
||||
if let Some(expires_at) = account.access_token_expires_at()
|
||||
&& expires_at <= now
|
||||
pub(crate) fn refresh_recovery_deadline(
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
) -> Option<SystemTime> {
|
||||
access_token_expires_at.and_then(|expires_at| expires_at.checked_add(REFRESH_RECOVERY_WINDOW))
|
||||
}
|
||||
|
||||
fn refresh_recovery_window_exhausted(
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
refresh_recovery_deadline(access_token_expires_at).is_some_and(|deadline| deadline <= now)
|
||||
}
|
||||
|
||||
fn bounded_refresh_retry_at(
|
||||
retry_at: SystemTime,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> SystemTime {
|
||||
let mut bounded = retry_at;
|
||||
if let Some(expires_at) = access_token_expires_at
|
||||
&& expires_at > observed_at
|
||||
&& bounded > expires_at
|
||||
{
|
||||
// AT 已过期时不能继续等待持久退避;后续额度/请求探测都必须先用 RT 续出可用 AT。
|
||||
return Some(expires_at);
|
||||
bounded = expires_at;
|
||||
}
|
||||
account.next_refresh_at().filter(|next| *next <= now)
|
||||
if let Some(deadline) = refresh_recovery_deadline(access_token_expires_at)
|
||||
&& bounded > deadline
|
||||
{
|
||||
bounded = deadline;
|
||||
}
|
||||
bounded
|
||||
}
|
||||
|
||||
fn oauth_refresh_retry_at(
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let retry_at = if attempt <= REFRESH_BACKOFF_MAX_ATTEMPTS {
|
||||
provider_refresh_backoff_at(account_id, observed_at, attempt, reason)?
|
||||
} else {
|
||||
provider_refresh_retry_at(account_id, observed_at, REFRESH_RECOVERY_DELAY, reason)?
|
||||
};
|
||||
Ok(bounded_refresh_retry_at(
|
||||
retry_at,
|
||||
access_token_expires_at,
|
||||
observed_at,
|
||||
))
|
||||
}
|
||||
|
||||
fn log_refresh_deferred(
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
retry_at: SystemTime,
|
||||
) {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
attempt,
|
||||
reason,
|
||||
retry_at = %DateTime::<Utc>::from(retry_at),
|
||||
access_token_expires_at = ?access_token_expires_at.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(access_token_expires_at)
|
||||
.map(DateTime::<Utc>::from),
|
||||
"OpenAI OAuth refresh deferred"
|
||||
);
|
||||
}
|
||||
|
||||
fn refresh_due_at(
|
||||
account: &ProviderAccount,
|
||||
policy: ProviderRefreshPolicy,
|
||||
now: SystemTime,
|
||||
) -> Option<SystemTime> {
|
||||
if refresh_recovery_window_exhausted(account.access_token_expires_at(), now) {
|
||||
// 即使 retry-not-before 异常地落在 deadline 之后,也必须进入终态化路径。
|
||||
return refresh_recovery_deadline(account.access_token_expires_at());
|
||||
}
|
||||
let access_token_expires_at = account.access_token_expires_at()?;
|
||||
if !policy.is_refresh_due(access_token_expires_at, now) {
|
||||
return None;
|
||||
}
|
||||
if account
|
||||
.next_refresh_at()
|
||||
.is_some_and(|retry_at| retry_at > now)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(access_token_expires_at)
|
||||
}
|
||||
|
||||
pub struct DueCodexCredential {
|
||||
pub account: ProviderAccount,
|
||||
pub secret: CodexOAuthSecret,
|
||||
principal: CodexCredentialPrincipal,
|
||||
installation_id: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DueCodexCredential {
|
||||
@@ -94,7 +179,6 @@ pub enum CodexCredentialRefreshError {
|
||||
pub struct CodexCredentialRefreshService {
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
identity: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -104,7 +188,6 @@ impl CodexCredentialRefreshService {
|
||||
pub fn new(
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
identity: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -112,7 +195,6 @@ impl CodexCredentialRefreshService {
|
||||
Self {
|
||||
repository,
|
||||
refresher,
|
||||
identity,
|
||||
leases,
|
||||
credential_state,
|
||||
runtime_policy,
|
||||
@@ -130,7 +212,7 @@ impl CodexCredentialRefreshService {
|
||||
excluded: &BTreeSet<ProviderAccountId>,
|
||||
) -> Result<Vec<CodexCredentialRefreshOutcome>, CodexCredentialRefreshError> {
|
||||
let policy = self.runtime_policy.load_refresh_policy().await?;
|
||||
let (due, mut outcomes) = self.list_due_refresh(excluded).await?;
|
||||
let (due, mut outcomes) = self.list_due_refresh(excluded, policy).await?;
|
||||
outcomes.reserve(due.len());
|
||||
let refreshed = stream::iter(due.into_iter().map(|credential| async move {
|
||||
let account_id = credential.account.id().to_string();
|
||||
@@ -145,7 +227,14 @@ impl CodexCredentialRefreshService {
|
||||
for (account_id, result) in refreshed {
|
||||
match result {
|
||||
Ok(outcome) => outcomes.push(outcome),
|
||||
Err(_) => outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id }),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
error = %error,
|
||||
"OpenAI OAuth refresh attempt failed"
|
||||
);
|
||||
outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(outcomes)
|
||||
@@ -157,6 +246,19 @@ impl CodexCredentialRefreshService {
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
let account_id = due.account.id().to_string();
|
||||
if refresh_recovery_window_exhausted(
|
||||
due.account.access_token_expires_at(),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
return self
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
"refresh_recovery_window_exhausted",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let capacity = self
|
||||
.leases
|
||||
.try_acquire(ProviderLeaseRequest::RefreshCapacity(
|
||||
@@ -187,12 +289,12 @@ impl CodexCredentialRefreshService {
|
||||
.as_ref()
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
match self.refresher.refresh(refresh_token.expose_secret()).await {
|
||||
Ok(tokens) => self.persist_success(due, tokens, policy).await,
|
||||
Ok(tokens) => self.persist_success(due, tokens).await,
|
||||
Err(RefreshFailure::InvalidGrant) => {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
"invalid_grant",
|
||||
"refresh_invalid_grant",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
.await
|
||||
@@ -234,6 +336,7 @@ impl CodexCredentialRefreshService {
|
||||
async fn list_due_refresh(
|
||||
&self,
|
||||
excluded: &BTreeSet<ProviderAccountId>,
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<
|
||||
(Vec<DueCodexCredential>, Vec<CodexCredentialRefreshOutcome>),
|
||||
CodexCredentialRefreshError,
|
||||
@@ -256,9 +359,10 @@ impl CodexCredentialRefreshService {
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
)
|
||||
&& refresh_due_at(account, now).is_some()
|
||||
&& refresh_due_at(account, policy, now).is_some()
|
||||
});
|
||||
accounts.sort_by_key(|account| (refresh_due_at(account, now), account.id().clone()));
|
||||
accounts
|
||||
.sort_by_key(|account| (refresh_due_at(account, policy, now), account.id().clone()));
|
||||
accounts.truncate(MAX_REFRESH_BATCH as usize);
|
||||
let mut due = Vec::with_capacity(accounts.len());
|
||||
let mut failures = Vec::new();
|
||||
@@ -274,15 +378,9 @@ impl CodexCredentialRefreshService {
|
||||
let Some(secret) = runtime.authentication.oauth() else {
|
||||
unreachable!("OAuth runtime authentication was checked above")
|
||||
};
|
||||
let Some(principal) = runtime.principal else {
|
||||
failures.push(CodexCredentialRefreshOutcome::Failed { account_id });
|
||||
continue;
|
||||
};
|
||||
due.push(DueCodexCredential {
|
||||
account,
|
||||
secret: secret.clone(),
|
||||
principal,
|
||||
installation_id: runtime.installation_id,
|
||||
});
|
||||
}
|
||||
Ok(_) | Err(_) => {
|
||||
@@ -297,116 +395,42 @@ impl CodexCredentialRefreshService {
|
||||
&self,
|
||||
due: DueCodexCredential,
|
||||
tokens: TokenPair,
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
if tokens.access_token.is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialRefreshError::InvalidRefreshResponse);
|
||||
}
|
||||
let refresh_token = tokens
|
||||
.refresh_token
|
||||
let TokenPair {
|
||||
access_token,
|
||||
refresh_token: rotated_refresh_token,
|
||||
expires_in,
|
||||
} = tokens;
|
||||
let refresh_token = rotated_refresh_token
|
||||
.map(SecretString::from)
|
||||
.or(due.secret.refresh_token);
|
||||
let account_id = due
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
let expectation = CodexIdentityExpectation::current(
|
||||
due.principal.oauth_subject,
|
||||
due.principal.poid,
|
||||
account_id.to_owned(),
|
||||
due.account.upstream_user_id().to_owned(),
|
||||
due.installation_id,
|
||||
)
|
||||
.map_err(|_| CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
access_token: SecretString::from(access_token),
|
||||
refresh_token,
|
||||
id_token: due.secret.id_token,
|
||||
};
|
||||
let verification = match self.identity.verify(&secret, &expectation).await {
|
||||
Ok(verification) => verification,
|
||||
Err(CodexIdentityVerificationError::Rejected) => {
|
||||
return self
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Invalid,
|
||||
"refreshed_identity_rejected",
|
||||
CodexCredentialRefreshOutcome::Invalidated {
|
||||
account_id: due.account.id().to_string(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Unavailable) => {
|
||||
// JWKS/签名边界短暂不可用属瞬态:保留现有凭据、推进退避重试,
|
||||
// 不终态失效账号。
|
||||
return if self
|
||||
.defer_refresh(&due.account, "identity-unavailable")
|
||||
.await?
|
||||
{
|
||||
Ok(CodexCredentialRefreshOutcome::Transient {
|
||||
account_id: due.account.id().to_string(),
|
||||
})
|
||||
} else {
|
||||
Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: due.account.id().to_string(),
|
||||
})
|
||||
};
|
||||
}
|
||||
};
|
||||
let profile = match verification {
|
||||
CodexIdentityVerification::Complete(profile) => profile,
|
||||
CodexIdentityVerification::SignedOnly(signed) => {
|
||||
let attempt = self
|
||||
.credential_state
|
||||
.record_refresh_backoff(due.account.id(), REFRESH_BACKOFF_WINDOW)
|
||||
.await
|
||||
.unwrap_or(1);
|
||||
let retry_at = provider_refresh_backoff_at(
|
||||
due.account.id(),
|
||||
SystemTime::now(),
|
||||
attempt,
|
||||
"identity-completion",
|
||||
)?;
|
||||
match self
|
||||
.repository
|
||||
.rotate_signed_secret(&due.account, secret, &signed, retry_at)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
return Ok(CodexCredentialRefreshOutcome::Transient {
|
||||
account_id: due.account.id().to_string(),
|
||||
});
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => {
|
||||
return Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: due.account.id().to_string(),
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
};
|
||||
let next_refresh_at = policy
|
||||
.next_attempt_at(
|
||||
due.account.id(),
|
||||
profile
|
||||
.access_token_expires_at
|
||||
.map(SystemTime::from)
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.map_err(CodexCredentialRefreshError::from)?;
|
||||
record_oauth_recovery(
|
||||
CodexOAuthRecoveryOperation::ScheduledRefresh,
|
||||
Some(due.account.id().as_str()),
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
let observed_at = SystemTime::now();
|
||||
let access_token_expires_at = expires_in.and_then(|value| observed_at.checked_add(value));
|
||||
// 成功轮换清掉仅用于失败退避的 retry-not-before;正常预刷新窗口由
|
||||
// worker 结合当前 runtime policy 动态判断,不写入账号时间字段。
|
||||
let next_refresh_at = None;
|
||||
let result = self
|
||||
.repository
|
||||
.rotate_oauth_secret(RotateCodexCredential {
|
||||
account_id: due.account.id().to_string(),
|
||||
expected_credential_revision: due.account.revision().get(),
|
||||
.rotate_refreshed_oauth_secret(
|
||||
&due.account,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
next_refresh_at: Some(next_refresh_at),
|
||||
})
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(revision) => {
|
||||
@@ -415,6 +439,13 @@ impl CodexCredentialRefreshService {
|
||||
.credential_state
|
||||
.clear_refresh_backoff(due.account.id())
|
||||
.await;
|
||||
tracing::info!(
|
||||
account_id = %due.account.id(),
|
||||
credential_revision = revision.get(),
|
||||
access_token_expires_at = ?access_token_expires_at.map(DateTime::<Utc>::from),
|
||||
refresh_retry_not_before = ?next_refresh_at.map(DateTime::<Utc>::from),
|
||||
"OpenAI OAuth refresh succeeded"
|
||||
);
|
||||
Ok(CodexCredentialRefreshOutcome::Refreshed {
|
||||
account_id: due.account.id().to_string(),
|
||||
credential_revision: revision.get(),
|
||||
@@ -447,16 +478,27 @@ impl CodexCredentialRefreshService {
|
||||
}
|
||||
match self
|
||||
.repository
|
||||
.apply_state(
|
||||
.apply_state_with_message(
|
||||
account,
|
||||
availability,
|
||||
Some(reason.to_owned()),
|
||||
None,
|
||||
SystemTime::now(),
|
||||
Some(reason.to_owned()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(outcome),
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
?availability,
|
||||
reason,
|
||||
access_token_expires_at = ?account.access_token_expires_at()
|
||||
.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.map(DateTime::<Utc>::from),
|
||||
"OpenAI OAuth refresh marked account terminal"
|
||||
);
|
||||
Ok(outcome)
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => {
|
||||
Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: account.id().to_string(),
|
||||
@@ -476,10 +518,24 @@ impl CodexCredentialRefreshService {
|
||||
.record_refresh_backoff(account.id(), REFRESH_BACKOFF_WINDOW)
|
||||
.await
|
||||
.unwrap_or(1);
|
||||
let retry_at =
|
||||
provider_refresh_backoff_at(account.id(), SystemTime::now(), attempt, reason)?;
|
||||
let retry_at = oauth_refresh_retry_at(
|
||||
account.id(),
|
||||
account.access_token_expires_at(),
|
||||
SystemTime::now(),
|
||||
attempt,
|
||||
reason,
|
||||
)?;
|
||||
match self.repository.defer_refresh(account, retry_at).await {
|
||||
Ok(_) => Ok(true),
|
||||
Ok(_) => {
|
||||
log_refresh_deferred(
|
||||
account.id(),
|
||||
account.access_token_expires_at(),
|
||||
attempt,
|
||||
reason,
|
||||
retry_at,
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => Ok(false),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
|
||||
@@ -6,18 +6,15 @@ use std::time::SystemTime;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
CredentialRevision, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::CodexSignedIdentity;
|
||||
use super::security::{CodexCredentialCodec, CodexCredentialDataError, CodexRuntimeCredential};
|
||||
use super::types::{
|
||||
CodexAccountProfile, CodexCredentialData, CodexOAuthSecret, RotateCodexCredential,
|
||||
};
|
||||
use super::types::{CodexCredentialData, CodexOAuthSecret, RotateCodexCredential};
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
|
||||
@@ -47,7 +44,6 @@ impl CodexCredentialRepository {
|
||||
.map_err(|_| CredentialRepositoryError::InvalidInput("credential_revision"))?;
|
||||
let current = self.store.load_credential(&account_id, expected).await?;
|
||||
let mut data = CodexCredentialCodec::decode_complete(¤t.credential)?;
|
||||
verify_identity(¤t, &data, &input.verified_account)?;
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
|
||||
@@ -83,13 +79,16 @@ impl CodexCredentialRepository {
|
||||
cas_revision(self.store.compare_and_swap_credential(update).await?)
|
||||
}
|
||||
|
||||
/// usage 暂时不可用时,只在签名 principal 与现有绑定完全一致后保存已轮换 token。
|
||||
pub async fn rotate_signed_secret(
|
||||
/// 持久化成功 RT exchange 的 token,同时保留既有账号身份投影。
|
||||
///
|
||||
/// Refresh endpoint 已经是这次 AT/RT 轮换的授权边界;这里仅以 revision
|
||||
/// CAS 保护并发写入,不重新验证新 access token 的身份声明。
|
||||
pub async fn rotate_refreshed_oauth_secret(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
secret: CodexOAuthSecret,
|
||||
signed: &CodexSignedIdentity,
|
||||
next_refresh_at: SystemTime,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
) -> Result<CredentialRevision, CredentialRepositoryError> {
|
||||
let current = self
|
||||
.store
|
||||
@@ -102,17 +101,6 @@ impl CodexCredentialRepository {
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
|
||||
if oauth.principal.oauth_subject != signed.oauth_subject()
|
||||
|| oauth.principal.poid.as_deref() != signed.poid()
|
||||
|| signed
|
||||
.claimed_account_id()
|
||||
.is_some_and(|value| account.upstream_account_id() != Some(value))
|
||||
|| signed
|
||||
.claimed_user_id()
|
||||
.is_some_and(|value| account.upstream_user_id() != value)
|
||||
{
|
||||
return Err(CredentialRepositoryError::IdentityMismatch);
|
||||
}
|
||||
oauth.access_token = secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = secret
|
||||
.refresh_token
|
||||
@@ -129,8 +117,8 @@ impl CodexCredentialRepository {
|
||||
unchanged_profile(account),
|
||||
credential,
|
||||
secret.refresh_token.is_some(),
|
||||
Some(SystemTime::from(signed.access_token_expires_at())),
|
||||
Some(next_refresh_at),
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
)
|
||||
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?;
|
||||
cas_revision(self.store.compare_and_swap_credential(update).await?)
|
||||
@@ -233,21 +221,34 @@ impl CodexCredentialRepository {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
reason: Option<String>,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
self.apply_state_with_message(account, availability, observed_at, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 带上游错误描述的状态写入;上游 message 优先,终态缺失 message 时写稳定兜底,
|
||||
/// 恢复 `Ready` 时清空。
|
||||
pub async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
message,
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -272,24 +273,6 @@ fn cas_revision(
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_identity(
|
||||
current: &LoadedCredential,
|
||||
credential: &CodexCredentialData,
|
||||
verified: &CodexAccountProfile,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| current.account.upstream_account_id() != Some(verified.chatgpt_account_id.as_str())
|
||||
|| verified.chatgpt_user_id != current.account.upstream_user_id()
|
||||
|| credential.oauth().is_none_or(|credential| {
|
||||
verified.oauth_subject != credential.principal.oauth_subject
|
||||
|| verified.poid != credential.principal.poid
|
||||
})
|
||||
{
|
||||
return Err(CredentialRepositoryError::IdentityMismatch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn required_time(value: Option<DateTime<Utc>>) -> Result<SystemTime, CredentialRepositoryError> {
|
||||
value
|
||||
.map(SystemTime::from)
|
||||
@@ -306,8 +289,6 @@ pub enum CredentialRepositoryError {
|
||||
InvalidInput(&'static str),
|
||||
#[error("Codex credential data is invalid")]
|
||||
InvalidCredentialData,
|
||||
#[error("Codex credential identity does not match")]
|
||||
IdentityMismatch,
|
||||
#[error("Codex credential revision conflict")]
|
||||
RevisionConflict,
|
||||
#[error("provider account store is unavailable")]
|
||||
|
||||
@@ -102,12 +102,32 @@ impl CodexCredentialCodec {
|
||||
account: &CodexAccountProfile,
|
||||
cookies: Vec<CodexCookie>,
|
||||
) -> Result<PlaintextCredential, CodexCredentialDataError> {
|
||||
Self::encode_complete(CodexCredentialData::OAuth(CodexOAuthCredentialData {
|
||||
schema_version: CODEX_CREDENTIAL_SCHEMA_VERSION,
|
||||
principal: CodexCredentialPrincipal {
|
||||
Self::encode_oauth(
|
||||
secret,
|
||||
Some(CodexCredentialPrincipal {
|
||||
oauth_subject: account.oauth_subject.clone(),
|
||||
poid: account.poid.clone(),
|
||||
},
|
||||
}),
|
||||
cookies,
|
||||
)
|
||||
}
|
||||
|
||||
/// 为尚未解析资料的 OAuth 账号编码凭据。
|
||||
pub(crate) fn encode_unresolved(
|
||||
secret: &CodexOAuthSecret,
|
||||
cookies: Vec<CodexCookie>,
|
||||
) -> Result<PlaintextCredential, CodexCredentialDataError> {
|
||||
Self::encode_oauth(secret, None, cookies)
|
||||
}
|
||||
|
||||
fn encode_oauth(
|
||||
secret: &CodexOAuthSecret,
|
||||
principal: Option<CodexCredentialPrincipal>,
|
||||
cookies: Vec<CodexCookie>,
|
||||
) -> Result<PlaintextCredential, CodexCredentialDataError> {
|
||||
Self::encode_complete(CodexCredentialData::OAuth(CodexOAuthCredentialData {
|
||||
schema_version: CODEX_CREDENTIAL_SCHEMA_VERSION,
|
||||
principal,
|
||||
installation_id: uuid::Uuid::new_v4().to_string(),
|
||||
access_token: secret.access_token.expose_secret().to_owned(),
|
||||
refresh_token: secret
|
||||
@@ -171,7 +191,7 @@ impl CodexCredentialCodec {
|
||||
refresh_token: data.refresh_token.map(SecretString::from),
|
||||
id_token: data.id_token.map(SecretString::from),
|
||||
}),
|
||||
Some(data.principal),
|
||||
data.principal,
|
||||
data.installation_id,
|
||||
data.cookies,
|
||||
data.oauth_client_id,
|
||||
@@ -232,9 +252,6 @@ impl CodexCredentialCodec {
|
||||
let existing = Self::decode_complete(existing)?;
|
||||
match (&mut incoming, existing) {
|
||||
(CodexCredentialData::OAuth(incoming), CodexCredentialData::OAuth(existing)) => {
|
||||
if incoming.principal != existing.principal {
|
||||
return Err(CodexCredentialDataError::Invalid);
|
||||
}
|
||||
incoming.installation_id = existing.installation_id;
|
||||
}
|
||||
(
|
||||
@@ -260,23 +277,7 @@ impl CodexCredentialCodec {
|
||||
fn validate(data: &CodexCredentialData) -> Result<(), CodexCredentialDataError> {
|
||||
let (installation_id, cookies) = match data {
|
||||
CodexCredentialData::OAuth(data) => {
|
||||
if data.schema_version != CODEX_CREDENTIAL_SCHEMA_VERSION
|
||||
|| !valid_identity(&data.principal.oauth_subject)
|
||||
|| data
|
||||
.principal
|
||||
.poid
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_identity(value))
|
||||
|| !valid_secret(&data.access_token)
|
||||
|| data
|
||||
.refresh_token
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_secret(value))
|
||||
|| data
|
||||
.id_token
|
||||
.as_deref()
|
||||
.is_some_and(|value| !valid_secret(value))
|
||||
{
|
||||
if data.schema_version != CODEX_CREDENTIAL_SCHEMA_VERSION {
|
||||
return Err(CodexCredentialDataError::Invalid);
|
||||
}
|
||||
(&data.installation_id, &data.cookies)
|
||||
|
||||
@@ -25,15 +25,14 @@ use super::agent_identity::CodexAgentIdentityTaskService;
|
||||
use super::catalog::CodexCredentialCatalogService;
|
||||
use super::cookie::CodexCookiePolicy;
|
||||
use super::quota::CodexCredentialQuotaService;
|
||||
use super::refresh::refresh_recovery_deadline;
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use super::security::CodexRuntimeAuthentication;
|
||||
use super::types::{
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CodexCookie, CodexCookieCaptureOutcome,
|
||||
RuntimeCodexCookie,
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CODEX_AUTHENTICATION_KIND_OAUTH, CodexCookie,
|
||||
CodexCookieCaptureOutcome, RuntimeCodexCookie,
|
||||
};
|
||||
|
||||
const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const CLOUDFLARE_PATH_BLOCK_COOLDOWN: Duration = Duration::from_secs(30);
|
||||
const CLOUDFLARE_RECOVERY_STALE_AFTER: Duration = Duration::from_secs(60 * 60);
|
||||
const CLOUDFLARE_CHALLENGE_BACKOFF: [Duration; 4] = [
|
||||
Duration::from_secs(10),
|
||||
@@ -75,6 +74,21 @@ pub enum CodexAccountFailure {
|
||||
CloudflarePathBlocked,
|
||||
}
|
||||
|
||||
impl CodexAccountFailure {
|
||||
/// 上游未提供可展示错误文本时,保留已分类的稳定原因。
|
||||
const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::CredentialExpired => Some("upstream_credential_expired"),
|
||||
Self::IdentityVerificationRequired => Some("upstream_identity_verification_required"),
|
||||
Self::Banned => Some("upstream_account_banned"),
|
||||
Self::QuotaExhausted => Some("upstream_quota_exhausted"),
|
||||
Self::UsageLimitExhausted { .. } => Some("upstream_usage_limit_exhausted"),
|
||||
Self::CloudflarePathBlocked => Some("upstream_cloudflare_path_blocked"),
|
||||
Self::RateLimited { .. } | Self::CloudflareChallenge { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RiskRecoveryState {
|
||||
challenge_count: u32,
|
||||
@@ -84,7 +98,6 @@ struct RiskRecoveryState {
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum CookieRecovery {
|
||||
None,
|
||||
ExpireAt(SystemTime),
|
||||
Clear,
|
||||
}
|
||||
@@ -297,6 +310,24 @@ impl CodexCredentialSelector {
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
|
||||
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
|
||||
let accounts = if diagnostic {
|
||||
accounts
|
||||
} else {
|
||||
let mut retained = Vec::with_capacity(accounts.len());
|
||||
for account in accounts {
|
||||
let cooling = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
if cooling.is_none() {
|
||||
retained.push(account);
|
||||
}
|
||||
}
|
||||
retained
|
||||
};
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -321,6 +352,7 @@ impl CodexCredentialSelector {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
})
|
||||
@@ -329,39 +361,6 @@ impl CodexCredentialSelector {
|
||||
AccountCandidate { account, signals }
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// 旧版开关只影响“候选资格”。保留原始账号供 revision-fenced 凭据加载,
|
||||
// 用影子候选承载放宽后的资格,避免伪造已持久化账号的运行状态。
|
||||
let quota_exhausted_candidates = (!self.skip_exhausted).then(|| {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut candidate| {
|
||||
candidate.account = quota_exhausted_scheduling_projection(&candidate.account);
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let scheduling_candidates = quota_exhausted_candidates.as_deref().unwrap_or(&candidates);
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
self.resolve_session_affinity(
|
||||
request.session_affinity_key,
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let cyber_policy_scope = self
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
.await;
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if let Some(state) = cyber_policy_scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.state.as_ref())
|
||||
{
|
||||
excluded.extend(state.excluded_accounts().iter().cloned());
|
||||
}
|
||||
let continuation_account = match request.attempt.continuation_attempt() {
|
||||
ContinuationAttempt::Native => request
|
||||
.attempt
|
||||
@@ -383,7 +382,74 @@ impl CodexCredentialSelector {
|
||||
{
|
||||
return Err(CredentialSelectionError::NoEligibleCredential);
|
||||
}
|
||||
let pinned_account = required_account.or(continuation_account);
|
||||
let pinned_account = required_account.or_else(|| continuation_account.clone());
|
||||
let preserve_native_continuation_state = matches!(
|
||||
request.attempt.continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
) && continuation_account.is_some();
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if diagnostic {
|
||||
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
|
||||
candidates.clone()
|
||||
} else if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut candidate| {
|
||||
// 已命中 previous_response_id 的 native continuation 先检查原账号。
|
||||
// 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态,
|
||||
// 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。
|
||||
if !preserve_native_continuation_state
|
||||
|| pinned_account.as_ref() != Some(candidate.account.id())
|
||||
{
|
||||
candidate.account =
|
||||
quota_exhausted_scheduling_projection(&candidate.account);
|
||||
}
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let scheduling_candidates = scheduling_candidates.as_slice();
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
self.resolve_session_affinity(
|
||||
request.session_affinity_key,
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let cyber_policy_scope = self
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
.await;
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if !diagnostic
|
||||
&& preserve_native_continuation_state
|
||||
&& let Some(pinned) = pinned_account.as_ref()
|
||||
&& candidates.iter().any(|candidate| {
|
||||
candidate.account.id() == pinned && candidate.signals.quota_limit_reached
|
||||
})
|
||||
{
|
||||
// `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的
|
||||
// native response handle 先将真实触顶状态交给 Coordinator,随后才可
|
||||
// 通过 ReplayAny 按当前调度策略换号。
|
||||
excluded.insert(pinned.clone());
|
||||
}
|
||||
if let Some(state) = cyber_policy_scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.state.as_ref())
|
||||
{
|
||||
excluded.extend(state.excluded_accounts().iter().cloned());
|
||||
}
|
||||
if let Some(required) = pinned_account.as_ref() {
|
||||
for candidate in &candidates {
|
||||
if candidate.account.id() != required {
|
||||
@@ -680,120 +746,99 @@ impl CodexCredentialSelector {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
failure: CodexAccountFailure,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let now = SystemTime::now();
|
||||
let (availability, reason, cooldown_until, cookie_recovery) = match failure {
|
||||
CodexAccountFailure::CredentialExpired => (
|
||||
AccountAvailability::Expired,
|
||||
Some("credential_expired".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::IdentityVerificationRequired => (
|
||||
AccountAvailability::Invalid,
|
||||
Some("identity_verification_required".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::Banned => (
|
||||
AccountAvailability::Banned,
|
||||
Some("account_banned".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::QuotaExhausted => (
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::UsageLimitExhausted { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(
|
||||
account,
|
||||
retry_after,
|
||||
"usage_limit_exhausted",
|
||||
now,
|
||||
)
|
||||
.await;
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| failure.fallback_error_reason().map(str::to_owned));
|
||||
match failure {
|
||||
CodexAccountFailure::CredentialExpired => {
|
||||
if account.authentication_kind() == CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
&& account.has_refresh_token()
|
||||
&& account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
&& refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.is_some_and(|deadline| deadline > now)
|
||||
{
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
access_token_expires_at = ?account.access_token_expires_at()
|
||||
.map(chrono::DateTime::<chrono::Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.map(chrono::DateTime::<chrono::Utc>::from),
|
||||
"OpenAI access token expired; retaining account for bounded OAuth refresh recovery"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::IdentityVerificationRequired => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::Banned => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
|
||||
.await
|
||||
}
|
||||
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
|
||||
CodexAccountFailure::QuotaExhausted
|
||||
| CodexAccountFailure::UsageLimitExhausted { .. } => {
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// 429:临时限流只写 quota,availability 不改变。
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(account, retry_after, "rate_limited", now)
|
||||
.await;
|
||||
self.quota
|
||||
.apply_rate_limit_429(account, retry_after, now)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
// Cloudflare 挑战:内存退避表(记录风险计数),不写 availability。
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after } => {
|
||||
let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after);
|
||||
let cooldown_until = now.checked_add(delay);
|
||||
(
|
||||
AccountAvailability::Cooldown,
|
||||
Some("cloudflare_challenge".to_owned()),
|
||||
cooldown_until,
|
||||
cooldown_until.map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt),
|
||||
)
|
||||
let recovery = now
|
||||
.checked_add(delay)
|
||||
.map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt);
|
||||
self.apply_cookie_recovery(account, recovery).await?;
|
||||
Ok(())
|
||||
}
|
||||
// Cloudflare 路径被封:连续超阈值才置 Invalid;否则内存退避。
|
||||
CodexAccountFailure::CloudflarePathBlocked => {
|
||||
let blocked = self.record_cloudflare_path_block(account.id(), now);
|
||||
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
|
||||
(
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::Invalid,
|
||||
Some("cloudflare_path_blocked".to_owned()),
|
||||
None,
|
||||
CookieRecovery::Clear,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
AccountAvailability::Cooldown,
|
||||
Some("cloudflare_path_blocked".to_owned()),
|
||||
now.checked_add(CLOUDFLARE_PATH_BLOCK_COOLDOWN),
|
||||
CookieRecovery::Clear,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
self.apply_cookie_recovery(account, CookieRecovery::Clear)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
self.repository
|
||||
.apply_state(account, availability, reason, cooldown_until, now)
|
||||
.await?;
|
||||
self.apply_cookie_recovery(account, cookie_recovery).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_resettable_limit_failure(
|
||||
async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
reason: &'static str,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let current = self.current_account(account.id()).await?;
|
||||
if current.revision() != account.revision() {
|
||||
return Err(CredentialRepositoryError::RevisionConflict.into());
|
||||
}
|
||||
let cooldown_until =
|
||||
observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN));
|
||||
let cooldown_until = match (current.availability(), current.cooldown_until()) {
|
||||
(AccountAvailability::QuotaExhausted, Some(existing)) => {
|
||||
cooldown_until.map_or(Some(existing), |until| Some(existing.max(until)))
|
||||
}
|
||||
_ => cooldown_until,
|
||||
};
|
||||
let reason = if current.availability() == AccountAvailability::QuotaExhausted
|
||||
&& current.cooldown_until().is_none()
|
||||
{
|
||||
Some("quota_exhausted".to_owned())
|
||||
} else {
|
||||
Some(reason.to_owned())
|
||||
};
|
||||
// 一次真实 429 证明上游已经接受当前凭据,可纠正旧的认证/封禁终态。
|
||||
// resettable 用量限制沿用 v2 的 quota lock 语义,并保留/延长复核时间。
|
||||
self.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.apply_state_with_message(account, availability, observed_at, message)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -857,7 +902,6 @@ impl CodexCredentialSelector {
|
||||
|| !matches!(
|
||||
current.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
@@ -867,13 +911,7 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.apply_state(¤t, AccountAvailability::Ready, SystemTime::now())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -985,15 +1023,11 @@ impl CodexCredentialSelector {
|
||||
account: &ProviderAccount,
|
||||
recovery: CookieRecovery,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
if matches!(recovery, CookieRecovery::None) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut data = self.repository.load_complete_data(account).await?;
|
||||
if data.cookies().is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
match recovery {
|
||||
CookieRecovery::None => return Ok(()),
|
||||
CookieRecovery::ExpireAt(expires_at) => {
|
||||
let expires_at = chrono::DateTime::<chrono::Utc>::from(expires_at);
|
||||
for cookie in data.cookies_mut() {
|
||||
@@ -1013,11 +1047,9 @@ impl CodexCredentialSelector {
|
||||
|
||||
fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount {
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
return account.clone().with_runtime_state(
|
||||
account.enabled(),
|
||||
AccountAvailability::Ready,
|
||||
account.cooldown_until(),
|
||||
);
|
||||
return account
|
||||
.clone()
|
||||
.with_runtime_state(account.enabled(), AccountAvailability::Ready);
|
||||
}
|
||||
account.clone()
|
||||
}
|
||||
@@ -1032,7 +1064,6 @@ fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> Af
|
||||
}
|
||||
match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountAvailability::Cooldown => AffinityEscapeReason::Cooldown,
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
@@ -1171,8 +1202,7 @@ impl From<CredentialRepositoryError> for CredentialSelectionError {
|
||||
fn from(error: CredentialRepositoryError) -> Self {
|
||||
match error {
|
||||
CredentialRepositoryError::InvalidInput(_)
|
||||
| CredentialRepositoryError::InvalidCredentialData
|
||||
| CredentialRepositoryError::IdentityMismatch => Self::InvalidCredential,
|
||||
| CredentialRepositoryError::InvalidCredentialData => Self::InvalidCredential,
|
||||
CredentialRepositoryError::RevisionConflict | CredentialRepositoryError::Store => {
|
||||
Self::Store
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ use serde::Deserialize;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
const MAX_TOKEN_LIFETIME_SECONDS: u64 = 366 * 24 * 60 * 60;
|
||||
const MAX_OAUTH_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
const TOKEN_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const TOKEN_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
@@ -24,7 +23,9 @@ pub const OFFICIAL_CODEX_REDIRECT_URI: &str = "http://localhost:1455/auth/callba
|
||||
pub struct TokenPair {
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
pub expires_in: Duration,
|
||||
/// OAuth endpoint 可以省略或返回零 expiry;这不影响已经成功的 token
|
||||
/// exchange,只意味着调用方不能据此安排下一次刷新。
|
||||
pub expires_in: Option<Duration>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for TokenPair {
|
||||
@@ -76,10 +77,14 @@ impl fmt::Debug for AuthorizationCodeGrant {
|
||||
}
|
||||
}
|
||||
|
||||
/// 官方 token endpoint 返回、尚待 OIDC/JWKS 绑定校验的 token set。
|
||||
/// 官方 token endpoint 返回的 token set。
|
||||
///
|
||||
/// 与官方首次 authorization-code exchange 一致:`id_token`、`access_token` 与
|
||||
/// `refresh_token` 都是响应的必填字段。这里不检查 token 内容、签名或 claims。
|
||||
pub struct AuthorizationTokenSet {
|
||||
pub secret: crate::credential::CodexOAuthSecret,
|
||||
pub id_token: SecretString,
|
||||
pub expires_in: Option<Duration>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AuthorizationTokenSet {
|
||||
@@ -88,6 +93,7 @@ impl fmt::Debug for AuthorizationTokenSet {
|
||||
.debug_struct("AuthorizationTokenSet")
|
||||
.field("secret", &"[REDACTED]")
|
||||
.field("id_token", &"[REDACTED]")
|
||||
.field("expires_in", &self.expires_in)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -168,16 +174,15 @@ pub fn official_openai_token_client() -> Result<OpenAiTokenClient, TokenClientBu
|
||||
struct RefreshTokenResponse {
|
||||
access_token: String,
|
||||
refresh_token: Option<String>,
|
||||
expires_in: u64,
|
||||
expires_in: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AuthorizationCodeResponse {
|
||||
access_token: String,
|
||||
refresh_token: Option<String>,
|
||||
refresh_token: String,
|
||||
id_token: String,
|
||||
token_type: String,
|
||||
expires_in: u64,
|
||||
expires_in: Option<u64>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -227,13 +232,6 @@ impl AuthorizationCodeExchanger for OpenAiTokenClient {
|
||||
AuthorizationCodeExchangeError::Ambiguous
|
||||
}
|
||||
})?;
|
||||
let success_is_json = response.status().is_success()
|
||||
&& response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.split(';').next())
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("application/json"));
|
||||
let (status, body) = read_bounded_response(response)
|
||||
.await
|
||||
.map_err(|_| AuthorizationCodeExchangeError::Ambiguous)?;
|
||||
@@ -243,30 +241,17 @@ impl AuthorizationCodeExchanger for OpenAiTokenClient {
|
||||
_ => AuthorizationCodeExchangeError::Rejected,
|
||||
});
|
||||
}
|
||||
if !success_is_json {
|
||||
return Err(AuthorizationCodeExchangeError::Rejected);
|
||||
}
|
||||
let tokens = serde_json::from_slice::<AuthorizationCodeResponse>(&body)
|
||||
.map_err(|_| AuthorizationCodeExchangeError::Rejected)?;
|
||||
if tokens.access_token.is_empty()
|
||||
|| tokens.id_token.is_empty()
|
||||
|| !tokens.token_type.eq_ignore_ascii_case("bearer")
|
||||
|| tokens.expires_in == 0
|
||||
|| tokens.expires_in > MAX_TOKEN_LIFETIME_SECONDS
|
||||
|| tokens
|
||||
.refresh_token
|
||||
.as_deref()
|
||||
.is_some_and(|token| token.is_empty())
|
||||
{
|
||||
return Err(AuthorizationCodeExchangeError::Rejected);
|
||||
}
|
||||
let id_token = SecretString::from(tokens.id_token);
|
||||
Ok(AuthorizationTokenSet {
|
||||
secret: crate::credential::CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
refresh_token: tokens.refresh_token.map(SecretString::from),
|
||||
refresh_token: Some(SecretString::from(tokens.refresh_token)),
|
||||
id_token: None,
|
||||
},
|
||||
id_token: SecretString::from(tokens.id_token),
|
||||
id_token,
|
||||
expires_in: optional_expiry(tokens.expires_in),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -294,19 +279,20 @@ async fn read_bounded_response(
|
||||
|
||||
fn parse_token_pair(body: &[u8]) -> Result<TokenPair, ()> {
|
||||
let tokens = serde_json::from_slice::<RefreshTokenResponse>(body).map_err(|_| ())?;
|
||||
if tokens.access_token.trim().is_empty() {
|
||||
return Err(());
|
||||
}
|
||||
if tokens.expires_in == 0 || tokens.expires_in > MAX_TOKEN_LIFETIME_SECONDS {
|
||||
return Err(());
|
||||
}
|
||||
Ok(TokenPair {
|
||||
access_token: tokens.access_token,
|
||||
// OAuth 刷新响应可能省略未变更的 RT;缺失时由调用方保留当前值。
|
||||
refresh_token: tokens.refresh_token,
|
||||
expires_in: Duration::from_secs(tokens.expires_in),
|
||||
expires_in: optional_expiry(tokens.expires_in),
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_expiry(value: Option<u64>) -> Option<Duration> {
|
||||
value
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.map(Duration::from_secs)
|
||||
}
|
||||
|
||||
fn classify_refresh_failure(status: StatusCode, body: &[u8]) -> RefreshFailure {
|
||||
// 5xx 与 429 按状态码判为瞬态:其正文(CDN/网关页等)不是权威 OAuth 错误响应,
|
||||
// 不据此终态失效账号。
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use secrecy::SecretString;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use url::Url;
|
||||
|
||||
/// OAuth AT/RT/ID Token;`Debug` 永不输出明文。
|
||||
@@ -29,7 +30,7 @@ impl fmt::Debug for CodexOAuthSecret {
|
||||
}
|
||||
}
|
||||
|
||||
/// 已由官方 JWT/OIDC 与认证 usage 响应共同确认的完整账号投影。
|
||||
/// OAuth credential 持久化时使用的账号投影。
|
||||
#[derive(Clone)]
|
||||
pub struct CodexAccountProfile {
|
||||
pub email: Option<String>,
|
||||
@@ -41,6 +42,116 @@ pub struct CodexAccountProfile {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 从 OAuth ID token payload 尽力提取的展示资料。
|
||||
///
|
||||
/// 这只是与官方客户端一致的 base64 JSON 读取,不校验 JWT 签名或 claims。
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct CodexOAuthMetadata {
|
||||
pub(crate) email: Option<String>,
|
||||
pub(crate) chatgpt_plan_type: Option<String>,
|
||||
pub(crate) chatgpt_user_id: Option<String>,
|
||||
pub(crate) chatgpt_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexOAuthMetadata {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CodexOAuthMetadata")
|
||||
.field("email", &self.email.as_ref().map(|_| "<redacted>"))
|
||||
.field(
|
||||
"chatgpt_user_id",
|
||||
&self.chatgpt_user_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field(
|
||||
"chatgpt_account_id",
|
||||
&self.chatgpt_account_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field("chatgpt_plan_type", &self.chatgpt_plan_type)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 按官方 `token_data.rs` 的字段优先级读取 ID token payload。
|
||||
///
|
||||
/// 与官方一致,`id_token` 的 JWT 外形、payload base64 或 JSON 无法解析时返回错误;
|
||||
/// 这不是签名或 claims 验证。各身份字段本身仍全部可缺失。
|
||||
pub(crate) fn parse_id_token_metadata(jwt: &str) -> Result<CodexOAuthMetadata, ()> {
|
||||
let claims = decode_jwt_payload::<IdTokenClaims>(jwt)?;
|
||||
let email = claims
|
||||
.email
|
||||
.or_else(|| claims.profile.and_then(|profile| profile.email));
|
||||
let Some(auth) = claims.auth else {
|
||||
return Ok(CodexOAuthMetadata {
|
||||
email,
|
||||
..CodexOAuthMetadata::default()
|
||||
});
|
||||
};
|
||||
Ok(CodexOAuthMetadata {
|
||||
email,
|
||||
chatgpt_plan_type: auth.chatgpt_plan_type,
|
||||
chatgpt_user_id: auth.chatgpt_user_id.or(auth.user_id),
|
||||
chatgpt_account_id: auth.chatgpt_account_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// 按官方 Codex 的方式,从 access token JWT payload 读取调度所需的过期时刻。
|
||||
///
|
||||
/// 此处只投影未验证的 `exp`,不把它当作身份或签名验证结果。无法解析或缺少
|
||||
/// `exp` 的直接导入仍可保留为没有已知过期时刻的 OAuth 凭据。
|
||||
pub(crate) fn parse_access_token_expiration(jwt: &str) -> Option<DateTime<Utc>> {
|
||||
let claims = decode_jwt_payload::<StandardJwtClaims>(jwt).ok()?;
|
||||
claims
|
||||
.exp
|
||||
.and_then(|seconds| DateTime::<Utc>::from_timestamp(seconds, 0))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct IdTokenClaims {
|
||||
#[serde(default)]
|
||||
email: Option<String>,
|
||||
#[serde(rename = "https://api.openai.com/profile", default)]
|
||||
profile: Option<ProfileClaims>,
|
||||
#[serde(rename = "https://api.openai.com/auth", default)]
|
||||
auth: Option<AuthClaims>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StandardJwtClaims {
|
||||
#[serde(default)]
|
||||
exp: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ProfileClaims {
|
||||
#[serde(default)]
|
||||
email: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AuthClaims {
|
||||
#[serde(default)]
|
||||
chatgpt_plan_type: Option<String>,
|
||||
#[serde(default)]
|
||||
chatgpt_user_id: Option<String>,
|
||||
#[serde(default)]
|
||||
user_id: Option<String>,
|
||||
#[serde(default)]
|
||||
chatgpt_account_id: Option<String>,
|
||||
}
|
||||
|
||||
fn decode_jwt_payload<T: DeserializeOwned>(jwt: &str) -> Result<T, ()> {
|
||||
let mut parts = jwt.split('.');
|
||||
let (Some(header), Some(payload), Some(signature)) = (parts.next(), parts.next(), parts.next())
|
||||
else {
|
||||
return Err(());
|
||||
};
|
||||
if header.is_empty() || payload.is_empty() || signature.is_empty() {
|
||||
return Err(());
|
||||
}
|
||||
let payload = URL_SAFE_NO_PAD.decode(payload).map_err(|_| ())?;
|
||||
serde_json::from_slice(&payload).map_err(|_| ())
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexAccountProfile {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
@@ -109,7 +220,8 @@ pub const CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY: &str = "agent_identity";
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CodexOAuthCredentialData {
|
||||
pub schema_version: u32,
|
||||
pub principal: CodexCredentialPrincipal,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub principal: Option<CodexCredentialPrincipal>,
|
||||
pub installation_id: String,
|
||||
pub access_token: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
||||
@@ -16,13 +16,10 @@ use gateway_core::task::WorkerContribution;
|
||||
use crate::admin::{OpenAiAdminProvider, OpenAiAdminServices, OpenAiOAuthPendingStore};
|
||||
use crate::credential::token_client::{AuthorizationCodeExchanger, TokenRefresher};
|
||||
use crate::credential::{
|
||||
CodexAccountIdentityService, CodexAccountIdentityVerifier, CodexAgentIdentityTaskService,
|
||||
CodexAuthenticatedAccountSource, CodexCookiePolicy, CodexCredentialAdmin,
|
||||
CodexAgentIdentityTaskService, CodexCookiePolicy, CodexCredentialAdmin,
|
||||
CodexCredentialAdminService, CodexCredentialCatalogService, CodexCredentialQuotaService,
|
||||
CodexCredentialRefreshService, CodexCredentialRepository, CodexCredentialSelector,
|
||||
CodexJwtIdentityVerifier, CodexOAuthAdmin, CodexOAuthAdminService, CodexSignedIdentityVerifier,
|
||||
OfficialCodexAgentIdentityTaskRegistrar, ReqwestCodexAuthenticatedAccountSource,
|
||||
ReqwestOpenAiJwksSource,
|
||||
CodexOAuthAdmin, CodexOAuthAdminService, OfficialCodexAgentIdentityTaskRegistrar,
|
||||
};
|
||||
use crate::transport::profile::{
|
||||
CodexCliReleaseService, CodexDesktopReleaseService, OfficialCodexCliReleaseTransport,
|
||||
@@ -82,7 +79,7 @@ pub async fn initialize(
|
||||
profile.clone(),
|
||||
Arc::new(
|
||||
OfficialCodexCliReleaseTransport::new()
|
||||
.map_err(|_| OpenAiInitializeError::DesktopRelease)?,
|
||||
.map_err(|_| OpenAiInitializeError::CliRelease)?,
|
||||
),
|
||||
));
|
||||
let repository = CodexCredentialRepository::new(Arc::clone(&accounts));
|
||||
@@ -111,6 +108,7 @@ pub async fn initialize(
|
||||
http.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
ports.cooldowns(),
|
||||
));
|
||||
let quota_skip_exhausted = config.quota_skip_exhausted();
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -141,34 +139,20 @@ pub async fn initialize(
|
||||
.map_err(OpenAiInitializeError::Provider)?
|
||||
.with_session_identity(session_identity),
|
||||
);
|
||||
|
||||
let token_client = Arc::new(
|
||||
credential::token_client::openai_token_client(config.token_client_config())
|
||||
.map_err(|_| OpenAiInitializeError::TokenClient)?,
|
||||
);
|
||||
let jwks = ReqwestOpenAiJwksSource::new().map_err(|_| OpenAiInitializeError::Identity)?;
|
||||
let signed: Arc<dyn CodexSignedIdentityVerifier> = Arc::new(
|
||||
CodexJwtIdentityVerifier::new(Box::new(jwks))
|
||||
.with_oauth_client_id(config.oauth_client_id()),
|
||||
);
|
||||
let account_source: Arc<dyn CodexAuthenticatedAccountSource> = Arc::new(
|
||||
ReqwestCodexAuthenticatedAccountSource::new(profile.clone(), config.base_url().to_owned())
|
||||
.map_err(|_| OpenAiInitializeError::Identity)?,
|
||||
);
|
||||
let identity: Arc<dyn CodexAccountIdentityVerifier> =
|
||||
Arc::new(CodexAccountIdentityService::new(signed, account_source));
|
||||
let refresher: Arc<dyn TokenRefresher> = token_client.clone();
|
||||
let exchanger: Arc<dyn AuthorizationCodeExchanger> = token_client;
|
||||
let credential_admin = Arc::new(CodexCredentialAdminService::new(
|
||||
Arc::clone(&refresher),
|
||||
Arc::clone(&identity),
|
||||
Arc::clone(&leases),
|
||||
Arc::clone(&runtime_policy),
|
||||
));
|
||||
let refresh = Arc::new(CodexCredentialRefreshService::new(
|
||||
repository,
|
||||
refresher,
|
||||
Arc::clone(&identity),
|
||||
Arc::clone(&leases),
|
||||
credential_state,
|
||||
Arc::clone(&runtime_policy),
|
||||
@@ -181,9 +165,7 @@ pub async fn initialize(
|
||||
CodexOAuthAdminService::new(
|
||||
pending,
|
||||
exchanger,
|
||||
Arc::clone(&identity),
|
||||
Arc::clone(&accounts),
|
||||
Arc::clone(&runtime_policy),
|
||||
CodexCredentialAdmin,
|
||||
)
|
||||
.with_oauth_client_id(config.oauth_client_id()),
|
||||
@@ -194,11 +176,9 @@ pub async fn initialize(
|
||||
accounts,
|
||||
OpenAiAdminServices {
|
||||
credentials: credential_admin,
|
||||
verifier: identity,
|
||||
oauth: oauth_admin,
|
||||
quota: Arc::clone("a),
|
||||
catalog: Arc::clone(&catalog),
|
||||
runtime_policy,
|
||||
},
|
||||
websocket_pool,
|
||||
desktop_release_status,
|
||||
@@ -257,8 +237,6 @@ pub enum OpenAiInitializeError {
|
||||
CookiePolicy,
|
||||
#[error("OpenAI token client could not initialize")]
|
||||
TokenClient,
|
||||
#[error("OpenAI identity verifier could not initialize")]
|
||||
Identity,
|
||||
#[error("OpenAI Agent Identity service could not initialize")]
|
||||
AgentIdentity,
|
||||
#[error("OpenAI credential administration could not initialize")]
|
||||
@@ -267,6 +245,8 @@ pub enum OpenAiInitializeError {
|
||||
Refresh,
|
||||
#[error("OpenAI Desktop release service could not initialize")]
|
||||
DesktopRelease,
|
||||
#[error("OpenAI CLI release service could not initialize")]
|
||||
CliRelease,
|
||||
#[error("OpenAI worker plan is invalid")]
|
||||
Worker,
|
||||
}
|
||||
|
||||
@@ -273,6 +273,8 @@ impl Provider for CodexProvider {
|
||||
.await
|
||||
.map_err(map_selection_error)?;
|
||||
let lease = Arc::new(lease);
|
||||
// 首字计时的起点:账号选择完成之后、上游建立之前。
|
||||
let output_started_at = Instant::now();
|
||||
let provider_kind = ProviderKind::new(PROVIDER_NAME)
|
||||
.map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent))?;
|
||||
let cross_account = context
|
||||
@@ -373,6 +375,7 @@ impl Provider for CodexProvider {
|
||||
catalog: Arc::clone(&self.catalog),
|
||||
agent_identity: Arc::clone(&self.agent_identity),
|
||||
lease: Arc::clone(&lease),
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_capture,
|
||||
});
|
||||
@@ -397,6 +400,7 @@ struct ColdResponse {
|
||||
catalog: Arc<CodexCredentialCatalogService>,
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
lease: Arc<CodexCredentialLease>,
|
||||
output_started_at: Instant,
|
||||
session_affinity_key: Option<ProviderSessionAffinityKey>,
|
||||
session_capture: Option<OpenAiSessionCapture>,
|
||||
}
|
||||
@@ -555,6 +559,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
catalog,
|
||||
agent_identity,
|
||||
lease,
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
mut session_capture,
|
||||
} = response;
|
||||
@@ -800,7 +805,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
capture.turn_state = Some(turn_state);
|
||||
}
|
||||
let first_event_changed =
|
||||
observation_state.observe_stream_chunk(&chunk, context.timing_started_at());
|
||||
observation_state.observe_stream_chunk(&chunk, output_started_at);
|
||||
let chunk_len = chunk.len();
|
||||
let (mut events, canonical_failure) = match decoder.push(&chunk) {
|
||||
CodexCanonicalOutcome::Events(events) => (events, None),
|
||||
@@ -830,7 +835,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
let timing_signals = decoder.take_timing_signals();
|
||||
let timing_changed = first_event_changed
|
||||
|| observation_state
|
||||
.observe_timing_signals(timing_signals, context.timing_started_at());
|
||||
.observe_timing_signals(timing_signals, output_started_at);
|
||||
let completed = events
|
||||
.iter()
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
@@ -909,7 +914,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
let service_tier_changed = observation_state
|
||||
.observe_upstream_service_tier(decoder.response_service_tier());
|
||||
let timing_changed = observation_state
|
||||
.observe_timing_signals(timing_signals, context.timing_started_at());
|
||||
.observe_timing_signals(timing_signals, output_started_at);
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
let rate_limits_changed = if updates.is_empty() {
|
||||
false
|
||||
@@ -1063,7 +1068,9 @@ impl OpenAiResponseObservationState {
|
||||
started_at: Instant,
|
||||
) -> bool {
|
||||
let mut changed = false;
|
||||
if signals.semantic_output {
|
||||
// 首个非前导输出事件(结构帧也算)开启首字计时;
|
||||
// 真实语义首字由 first_reasoning_ms / first_text_ms 单独观测。
|
||||
if signals.output_start {
|
||||
changed |= insert_first_timing(&mut self.timings.first_token_ms, started_at);
|
||||
}
|
||||
if signals.reasoning_output {
|
||||
@@ -1697,9 +1704,13 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,随 availability 状态一并持久化。
|
||||
error_message: Option<String>,
|
||||
cyber_policy_failure: bool,
|
||||
set_cookie_headers: Vec<String>,
|
||||
rate_limit_headers: Vec<(String, String)>,
|
||||
/// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。
|
||||
confirmed_exhaustion_reset_at: Option<i64>,
|
||||
observation: Option<ProviderResponseObservation>,
|
||||
capture_response_cookies: bool,
|
||||
}
|
||||
@@ -1709,9 +1720,11 @@ impl MappedProviderFailure {
|
||||
Self {
|
||||
error,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
set_cookie_headers: Vec::new(),
|
||||
rate_limit_headers: Vec::new(),
|
||||
confirmed_exhaustion_reset_at: None,
|
||||
observation: None,
|
||||
capture_response_cookies: false,
|
||||
}
|
||||
@@ -1869,6 +1882,18 @@ async fn apply_failure(
|
||||
failure.account_failure,
|
||||
Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. })
|
||||
);
|
||||
if needs_authoritative_quota_refresh
|
||||
&& let Err(error) = context
|
||||
.quota
|
||||
.record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"Failed to project confirmed OpenAI quota exhaustion"
|
||||
);
|
||||
}
|
||||
if failure.cyber_policy_failure {
|
||||
context
|
||||
.selector
|
||||
@@ -1882,7 +1907,7 @@ async fn apply_failure(
|
||||
.await;
|
||||
if let Err(error) = context
|
||||
.selector
|
||||
.record_failure(account, account_failure)
|
||||
.record_failure(account, account_failure, failure.error_message.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -2162,9 +2187,11 @@ fn map_upstream_failure(
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
account_failure: account_failure(category, failure.retry_after_seconds),
|
||||
error_message: failure.client_message,
|
||||
cyber_policy_failure,
|
||||
set_cookie_headers: failure.set_cookie_headers,
|
||||
rate_limit_headers: failure.rate_limit_headers,
|
||||
confirmed_exhaustion_reset_at: failure.usage_limit_resets_at,
|
||||
observation,
|
||||
capture_response_cookies: !matches!(
|
||||
category,
|
||||
@@ -2415,11 +2442,10 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let outcomes = self
|
||||
.service
|
||||
.refresh_due()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("OpenAI OAuth refresh failed"))?;
|
||||
let outcomes = self.service.refresh_due().await.map_err(|error| {
|
||||
tracing::error!(error = %error, "OpenAI OAuth refresh cycle failed");
|
||||
WorkerTaskError::safe("OpenAI OAuth refresh failed")
|
||||
})?;
|
||||
let mut refreshed = 0_u64;
|
||||
let mut invalidated = 0_u64;
|
||||
let mut banned = 0_u64;
|
||||
@@ -2448,6 +2474,18 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !outcomes.is_empty() {
|
||||
tracing::info!(
|
||||
refreshed,
|
||||
invalidated,
|
||||
banned,
|
||||
transient,
|
||||
lease_unavailable,
|
||||
stale,
|
||||
failed,
|
||||
"OpenAI OAuth refresh cycle completed"
|
||||
);
|
||||
}
|
||||
if transient > 0 || failed > 0 {
|
||||
tracing::warn!(
|
||||
refreshed,
|
||||
@@ -2533,7 +2571,6 @@ impl ScheduledTask for OpenAiQuotaTask {
|
||||
updated = summary.updated,
|
||||
exhausted = summary.exhausted,
|
||||
banned = summary.banned,
|
||||
cooldown = summary.cooldown,
|
||||
transient = summary.transient,
|
||||
stale = summary.stale,
|
||||
"OpenAI quota cycle contained operational failures"
|
||||
|
||||
@@ -270,6 +270,11 @@ impl CodexCanonicalDecoder {
|
||||
.event
|
||||
.as_deref()
|
||||
.or_else(|| value.get("type").and_then(Value::as_str));
|
||||
if event_type == Some("codex.rate_limits") {
|
||||
// HTTP transport has already projected this control frame into local quota facts.
|
||||
// It must not become client output or start first-output timing.
|
||||
return Ok(());
|
||||
}
|
||||
self.observe_response_service_tier(&value);
|
||||
let signals = response_event_signals(event_type, &value);
|
||||
self.merge_timing_signals(signals);
|
||||
@@ -310,6 +315,7 @@ impl CodexCanonicalDecoder {
|
||||
|
||||
fn merge_timing_signals(&mut self, signals: ResponseEventSignals) {
|
||||
self.timing_signals.protocol_progress |= signals.protocol_progress;
|
||||
self.timing_signals.output_start |= signals.output_start;
|
||||
self.timing_signals.semantic_output |= signals.semantic_output;
|
||||
self.timing_signals.reasoning_output |= signals.reasoning_output;
|
||||
self.timing_signals.text_output |= signals.text_output;
|
||||
@@ -410,7 +416,7 @@ impl CodexCanonicalDecoder {
|
||||
self.complete(event_type, value, output)
|
||||
}
|
||||
"response.failed" | "error" => Err(protocol_error_marker()),
|
||||
"response.rate_limits.updated" | "codex.rate_limits" | "response.metadata" => Ok(()),
|
||||
"response.rate_limits.updated" | "response.metadata" => Ok(()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,8 +87,13 @@ impl CodexBackendClient {
|
||||
let headers = self.request_headers_for_http_response(upstream_request, context)?;
|
||||
let headers_started_at = Instant::now();
|
||||
// 与官方 Codex app-server 一致:/v1/responses 请求体默认 zstd 压缩。
|
||||
// Codex 上游只交付 SSE;即使下游请求 `stream: false`,也要上游流式执行,
|
||||
// 再由 API 层收集 canonical events 并返回完整 JSON。不能把下游的传输偏好
|
||||
// 直接透传给 Codex,否则上游会以 400 拒绝非流式请求。
|
||||
let mut upstream_body = upstream_request.body().clone();
|
||||
upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true));
|
||||
let body =
|
||||
serde_json::to_vec(&upstream_request).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
|
||||
.map_err(CodexClientError::RequestCompression)?;
|
||||
let response = self
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use super::client::CodexClientVisibleUpstreamResponse;
|
||||
use super::protocol::responses::ResponsesSseFailure;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{StatusCode, header::HeaderMap};
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -113,6 +114,8 @@ pub struct CodexUpstreamFailure {
|
||||
pub(crate) client_error_type: Option<String>,
|
||||
pub(crate) client_response: Option<Box<CodexClientVisibleUpstreamResponse>>,
|
||||
pub(crate) identity_error_code: Option<String>,
|
||||
/// 官方 `usage_limit_reached` 错误体携带的窗口重置时间(Unix 秒)。
|
||||
pub(crate) usage_limit_resets_at: Option<i64>,
|
||||
pub(crate) retry_after_seconds: Option<u64>,
|
||||
pub(crate) request_id: Option<String>,
|
||||
pub(crate) set_cookie_headers: Vec<String>,
|
||||
@@ -139,6 +142,7 @@ impl CodexUpstreamFailure {
|
||||
) -> Self {
|
||||
let fields = ParsedUpstreamError::from_body(body);
|
||||
let category = classify_upstream_failure(
|
||||
UpstreamFailureSource::HttpResponse,
|
||||
Some(status),
|
||||
body,
|
||||
&fields,
|
||||
@@ -153,6 +157,9 @@ impl CodexUpstreamFailure {
|
||||
client_error_type: fields.error_type,
|
||||
client_response: client_response.cloned().map(Box::new),
|
||||
identity_error_code: diagnostics.identity_error_code.clone(),
|
||||
usage_limit_resets_at: (category == CodexFailureCategory::UsageLimitExhausted)
|
||||
.then_some(fields.resets_at)
|
||||
.flatten(),
|
||||
retry_after_seconds,
|
||||
request_id: diagnostics.request_id.clone(),
|
||||
set_cookie_headers: set_cookie_headers.to_vec(),
|
||||
@@ -174,11 +181,13 @@ impl CodexUpstreamFailure {
|
||||
error_type: failure.upstream_type.clone(),
|
||||
message: failure.message.clone(),
|
||||
client_message: Some(failure.message.clone()),
|
||||
resets_at: None,
|
||||
};
|
||||
let status = failure
|
||||
.explicit_status_code
|
||||
.and_then(|code| StatusCode::from_u16(code).ok());
|
||||
let category = classify_upstream_failure(
|
||||
UpstreamFailureSource::SseFailure,
|
||||
status,
|
||||
"",
|
||||
&fields,
|
||||
@@ -193,6 +202,7 @@ impl CodexUpstreamFailure {
|
||||
client_error_type: fields.error_type,
|
||||
client_response: None,
|
||||
identity_error_code: diagnostics.identity_error_code.clone(),
|
||||
usage_limit_resets_at: None,
|
||||
retry_after_seconds: failure.retry_after_seconds,
|
||||
request_id: diagnostics.request_id.clone(),
|
||||
set_cookie_headers: set_cookie_headers.to_vec(),
|
||||
@@ -246,6 +256,7 @@ struct ParsedUpstreamError {
|
||||
error_type: Option<String>,
|
||||
message: String,
|
||||
client_message: Option<String>,
|
||||
resets_at: Option<i64>,
|
||||
}
|
||||
|
||||
impl ParsedUpstreamError {
|
||||
@@ -256,6 +267,7 @@ impl ParsedUpstreamError {
|
||||
error_type: None,
|
||||
message: body.to_owned(),
|
||||
client_message: None,
|
||||
resets_at: None,
|
||||
};
|
||||
};
|
||||
let error = value
|
||||
@@ -282,15 +294,26 @@ impl ParsedUpstreamError {
|
||||
.clone()
|
||||
.or_else(|| error.as_str().and_then(non_empty_owned))
|
||||
.unwrap_or_else(|| body.to_owned());
|
||||
let resets_at = error
|
||||
.get("resets_at")
|
||||
.and_then(Value::as_i64)
|
||||
.filter(|seconds| DateTime::<Utc>::from_timestamp(*seconds, 0).is_some());
|
||||
Self {
|
||||
code,
|
||||
error_type,
|
||||
message,
|
||||
client_message,
|
||||
resets_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum UpstreamFailureSource {
|
||||
HttpResponse,
|
||||
SseFailure,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CodexUpstreamDiagnostics {
|
||||
pub status_code: Option<u16>,
|
||||
@@ -363,6 +386,7 @@ fn decode_identity_error_code(encoded: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
fn classify_upstream_failure(
|
||||
source: UpstreamFailureSource,
|
||||
status: Option<StatusCode>,
|
||||
body: &str,
|
||||
fields: &ParsedUpstreamError,
|
||||
@@ -376,6 +400,19 @@ fn classify_upstream_failure(
|
||||
let message = fields.message.to_ascii_lowercase();
|
||||
let body = body.to_ascii_lowercase();
|
||||
|
||||
// 与官方 Codex HTTP/WS 路径一致:429 只有结构化
|
||||
// `error.type=usage_limit_reached` 才能确认额度窗口耗尽;其余 429 都是临时限流。
|
||||
// SSE `response.failed` 的结构化字段形态不同,保留其 code/type 语义单独分类。
|
||||
if matches!(source, UpstreamFailureSource::HttpResponse)
|
||||
&& status == Some(StatusCode::TOO_MANY_REQUESTS)
|
||||
{
|
||||
return if error_type == "usage_limit_reached" {
|
||||
CodexFailureCategory::UsageLimitExhausted
|
||||
} else {
|
||||
CodexFailureCategory::RateLimited
|
||||
};
|
||||
}
|
||||
|
||||
if [code.as_str(), message.as_str(), body.as_str()]
|
||||
.into_iter()
|
||||
.any(is_model_unsupported)
|
||||
|
||||
@@ -14,21 +14,22 @@ use url::Url;
|
||||
pub const CODEX_DESKTOP_APPCAST_URL: &str =
|
||||
"https://persistent.oaistatic.com/codex-app-prod/appcast.xml";
|
||||
pub const CODEX_CLI_RELEASE_URL: &str = "https://registry.npmjs.org/@openai%2Fcodex/latest";
|
||||
/// 官方制品画像检查周期。
|
||||
/// 官方发布元数据检查周期。
|
||||
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
const APPCAST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_APPCAST_BYTES: usize = 1024 * 1024;
|
||||
const MAX_CLI_RELEASE_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Codex Desktop 上游请求身份。
|
||||
///
|
||||
/// 启动配置提供经源码审计的 Core、运行环境和初始 Desktop 版本。官方 appcast
|
||||
/// 检查成功后,会仅同步 Desktop 版本和构建号。
|
||||
/// 启动配置提供经源码审计的 Core、运行环境和 Desktop 启动版本。npm CLI metadata
|
||||
/// 与官方 appcast 检查成功后,分别同步 Core 版本和 Desktop 版本及构建号。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CodexWireProfile {
|
||||
/// `originator` 请求头及 User-Agent 产品名。
|
||||
pub originator: String,
|
||||
/// bundled Codex Core 版本;同时用于 `/codex/models?client_version=`。
|
||||
/// npm `@openai/codex` 启动版本;用于 `/codex/models?client_version=` 与 UA。
|
||||
pub codex_version: String,
|
||||
/// Desktop 应用版本,用于 app-server `clientInfo.version` 对应的 UA 后缀。
|
||||
pub desktop_version: String,
|
||||
@@ -47,16 +48,18 @@ pub struct CodexWireProfile {
|
||||
}
|
||||
|
||||
impl CodexWireProfile {
|
||||
/// 按 Codex Core 的官方格式生成最终 User-Agent。
|
||||
/// 按 bundled Core app-server 的官方格式生成最终 User-Agent。
|
||||
pub fn user_agent(&self) -> String {
|
||||
format!(
|
||||
"{}/{} ({} {}; {}) {}",
|
||||
"{}/{} ({} {}; {}) {} ({}; {})",
|
||||
self.originator,
|
||||
self.codex_version,
|
||||
self.os_type,
|
||||
self.os_version,
|
||||
self.arch,
|
||||
self.terminal,
|
||||
self.originator,
|
||||
self.desktop_version,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -96,7 +99,7 @@ impl CodexWireProfileState {
|
||||
profile.desktop_build = desktop_build.to_owned();
|
||||
}
|
||||
|
||||
/// 使用官方 npm metadata 发布的 Codex CLI 版本更新运行时 Core 画像。
|
||||
/// 使用官方 npm metadata 发布的 Codex CLI 版本更新运行时请求画像。
|
||||
pub fn update_cli_release(&self, codex_version: &str) {
|
||||
let mut profile = self
|
||||
.profile
|
||||
@@ -143,12 +146,12 @@ impl CodexCliReleaseTransport for OfficialCodexCliReleaseTransport {
|
||||
}
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|size| size > 64 * 1024)
|
||||
.is_some_and(|size| size > MAX_CLI_RELEASE_BYTES as u64)
|
||||
{
|
||||
return Err(CodexCliReleaseError::ResponseTooLarge);
|
||||
}
|
||||
let bytes = response.bytes().await?;
|
||||
if bytes.len() > 64 * 1024 {
|
||||
if bytes.len() > MAX_CLI_RELEASE_BYTES {
|
||||
return Err(CodexCliReleaseError::ResponseTooLarge);
|
||||
}
|
||||
let document: serde_json::Value = serde_json::from_slice(&bytes)
|
||||
|
||||
@@ -182,11 +182,13 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
|
||||
|
||||
/// 单个 Responses 事件对计时系统提供的稳定语义信号。
|
||||
///
|
||||
/// `protocol_progress` 只说明上游仍在工作,不能替代首字;其余字段分别标记
|
||||
/// 客户端可消费的输出、reasoning 输出与正文输出。
|
||||
/// `protocol_progress` 只说明上游仍在工作,不能替代首字;`output_start` 标记首个会
|
||||
/// 开启客户端输出的非前导事件(结构帧也算);其余字段分别标记客户端可消费的
|
||||
/// 输出、reasoning 输出与正文输出。
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct ResponseEventSignals {
|
||||
pub protocol_progress: bool,
|
||||
pub output_start: bool,
|
||||
pub semantic_output: bool,
|
||||
pub reasoning_output: bool,
|
||||
pub text_output: bool,
|
||||
@@ -194,13 +196,22 @@ pub struct ResponseEventSignals {
|
||||
|
||||
/// 从已解析的 Responses 事件提取计时语义。
|
||||
///
|
||||
/// 生命周期帧和结构帧不会被当作输出。`response.output_item.done` 与终态帧只有
|
||||
/// 实际携带文本、工具参数、推理或图片结果时才算语义输出。
|
||||
/// `output_start` 在任意非前导、非失败事件上置位(含结构帧
|
||||
/// `response.output_item.added`/`content_part.added`),用于开启首字计时;前导帧
|
||||
/// (`response.created`/`response.in_progress`)与失败帧不置位。语义输出仍要求
|
||||
/// 实际携带文本、工具参数、推理或图片结果,`response.output_item.done` 与终态帧
|
||||
/// 只有携带语义内容才算。
|
||||
pub fn response_event_signals(event_type: Option<&str>, value: &Value) -> ResponseEventSignals {
|
||||
let mut signals = ResponseEventSignals {
|
||||
protocol_progress: !matches!(event_type, Some("response.failed" | "error")),
|
||||
..ResponseEventSignals::default()
|
||||
};
|
||||
signals.output_start = event_type.is_some_and(|event_type| {
|
||||
!matches!(
|
||||
event_type,
|
||||
"response.created" | "response.in_progress" | "response.failed" | "error"
|
||||
)
|
||||
});
|
||||
match event_type {
|
||||
Some("response.output_text.delta") => {
|
||||
signals.text_output = non_empty_string(value.get("delta"));
|
||||
@@ -606,6 +617,26 @@ impl CodexResponsesRequest {
|
||||
)
|
||||
}
|
||||
|
||||
/// 返回会话亲和所用的子代理区分值。
|
||||
///
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
|
||||
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
|
||||
/// 传递给上游。该可选值仅参与会话亲和键派生。
|
||||
pub fn subagent_kind(&self) -> Option<String> {
|
||||
self.semantics()
|
||||
.subagent_kind
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| {
|
||||
self.client_metadata()
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
/// 设置 / 合并 client metadata。
|
||||
pub fn set_client_metadata(&mut self, client_metadata: Option<Value>) {
|
||||
match client_metadata {
|
||||
|
||||
@@ -10,6 +10,8 @@ use sha2::{Digest, Sha256};
|
||||
use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
const THREAD_SPAWN_SUBAGENT_KIND: &str = "thread_spawn";
|
||||
const THREAD_SPAWN_CONVERSATION_PREFIX: &str = "thread-spawn:";
|
||||
|
||||
const CROSS_ACCOUNT_IDENTITY_KEYS: &[&str] = &[
|
||||
"authorization",
|
||||
@@ -160,28 +162,52 @@ fn string_value(value: Option<&Value>) -> Option<String> {
|
||||
pub(crate) fn derive_conversation_anchor(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<(&'static str, String)> {
|
||||
thread_spawn_conversation_anchor(request).or_else(|| {
|
||||
request
|
||||
.client_session_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("thread", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
})
|
||||
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
|
||||
})
|
||||
}
|
||||
|
||||
/// A `thread_spawn` shares its parent session but is a separate child execution.
|
||||
/// Its child thread must therefore own the affinity and WebSocket pool identity.
|
||||
fn thread_spawn_conversation_anchor(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<(&'static str, String)> {
|
||||
if request.subagent_kind().as_deref() != Some(THREAD_SPAWN_SUBAGENT_KIND) {
|
||||
return None;
|
||||
}
|
||||
|
||||
request
|
||||
.client_session_id
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
.map(str::trim)
|
||||
.filter(|thread_id| !thread_id.is_empty())
|
||||
.map(|thread_id| {
|
||||
(
|
||||
"subagent-thread",
|
||||
format!("{THREAD_SPAWN_CONVERSATION_PREFIX}{thread_id}"),
|
||||
)
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("thread", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
})
|
||||
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
|
||||
}
|
||||
|
||||
const LEADING_SYSTEM_REMINDER_OPEN: &str = "<system-reminder>";
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use gateway_protocol::openai::events::is_rate_limit_header_name;
|
||||
use gateway_protocol::openai::events::{is_codex_quota_header_name, is_rate_limit_header_name};
|
||||
use reqwest::header::{HeaderMap, SET_COOKIE};
|
||||
|
||||
use super::diagnostics::CodexUpstreamDiagnostics;
|
||||
@@ -155,6 +155,9 @@ fn client_response_header_is_forwardable(name: &str, connection_options: &[Strin
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if is_codex_quota_header_name(&name) {
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name.as_str(),
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::{
|
||||
WebSocketPoolConnectLease, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
|
||||
WebSocketPoolLease,
|
||||
},
|
||||
pump::{PumpKeepalive, PumpedWebSocket},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket},
|
||||
};
|
||||
|
||||
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
|
||||
@@ -203,10 +203,12 @@ async fn prepare_unpooled_websocket(
|
||||
decision_started_at: Instant,
|
||||
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
|
||||
let permit = acquire_breaker_permit(breaker, origin_key)?;
|
||||
let context = pump_log_context_from_connection(request.connection());
|
||||
let connected = connect_with_budget(
|
||||
request.connection(),
|
||||
PumpKeepalive::disabled(),
|
||||
fast_path_budget,
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
let (connection, connect_elapsed) = finish_breaker_attempt(permit, connected)?;
|
||||
@@ -302,6 +304,10 @@ fn start_pooled_websocket_connect(
|
||||
let cancellation = connect_lease.cancellation_token();
|
||||
let fast_path_reporter = permit.fast_path_reporter();
|
||||
let keepalive = pool.keepalive();
|
||||
let context = PumpLogContext::new(
|
||||
Some(task_key.account_id().to_owned()),
|
||||
Some(task_key.conversation_id_hash()),
|
||||
);
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
pool.spawn_connect_task(async move {
|
||||
let connected = tokio::select! {
|
||||
@@ -319,7 +325,7 @@ fn start_pooled_websocket_connect(
|
||||
);
|
||||
return;
|
||||
}
|
||||
result = connect_with_budget(&connection, keepalive, None) => result,
|
||||
result = connect_with_budget(&connection, keepalive, None, context) => result,
|
||||
};
|
||||
match finish_breaker_attempt(permit, connected) {
|
||||
Ok((connection, connect_elapsed)) => {
|
||||
@@ -392,17 +398,31 @@ fn duration_millis_u64(duration: Duration) -> u64 {
|
||||
.max(1)
|
||||
}
|
||||
|
||||
/// 非池化建连的 pump 日志上下文:从业务头提取账号归属(会话 hash 只有池 key 才有)。
|
||||
fn pump_log_context_from_connection(connection: &CodexWebSocketConnection) -> PumpLogContext {
|
||||
let account_id = connection
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("chatgpt-account-id"))
|
||||
.map(|(_, value)| value.clone());
|
||||
PumpLogContext::new(account_id, None)
|
||||
}
|
||||
|
||||
async fn connect_with_budget(
|
||||
connection: &CodexWebSocketConnection,
|
||||
keepalive: PumpKeepalive,
|
||||
fast_path_budget: Option<Duration>,
|
||||
context: PumpLogContext,
|
||||
) -> Result<(PooledWebSocketConnection, Duration), CodexWebSocketExchangeError> {
|
||||
let started_at = Instant::now();
|
||||
let connected = match fast_path_budget {
|
||||
Some(budget) => timeout(budget, connect_pumped_websocket(connection, keepalive))
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
|
||||
None => connect_pumped_websocket(connection, keepalive).await,
|
||||
Some(budget) => timeout(
|
||||
budget,
|
||||
connect_pumped_websocket(connection, keepalive, context),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
|
||||
None => connect_pumped_websocket(connection, keepalive, context).await,
|
||||
}?;
|
||||
let (websocket, response) = connected;
|
||||
Ok((
|
||||
|
||||
@@ -30,7 +30,7 @@ use super::{
|
||||
error::CodexWebSocketExchangeError,
|
||||
model::{CodexWebSocketConnection, CodexWebSocketRequest, WebSocketContinuationRequirement},
|
||||
pool::CodexWebSocketConnectionMetadata,
|
||||
pump::{PumpKeepalive, PumpedWebSocket, RawWsStream},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket, RawWsStream},
|
||||
};
|
||||
|
||||
const WEBSOCKET_EXTENSIONS: &str = "permessage-deflate; client_max_window_bits";
|
||||
@@ -93,9 +93,10 @@ pub fn responses_websocket_endpoint(base_url: &str) -> String {
|
||||
pub(super) async fn connect_pumped_websocket(
|
||||
connection: &CodexWebSocketConnection,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
) -> Result<(PumpedWebSocket, WsResponse<Option<Vec<u8>>>), CodexWebSocketExchangeError> {
|
||||
let (raw, response) = connect_websocket(connection).await?;
|
||||
Ok((PumpedWebSocket::new(raw, keepalive), response))
|
||||
Ok((PumpedWebSocket::new(raw, keepalive, context), response))
|
||||
}
|
||||
|
||||
pub(super) async fn send_websocket_request(
|
||||
|
||||
@@ -36,6 +36,22 @@ pub(crate) type RawWsStream = WebSocketStream<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
const PUMP_COMMAND_BUFFER: usize = 32;
|
||||
|
||||
/// pump 日志的账号级关联上下文;空闲连接被上游重置等场景用于回溯归属。
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct PumpLogContext {
|
||||
pub(crate) account_id: Option<String>,
|
||||
pub(crate) conversation_id_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl PumpLogContext {
|
||||
pub(crate) fn new(account_id: Option<String>, conversation_id_hash: Option<String>) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
conversation_id_hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// pump 保活策略。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct PumpKeepalive {
|
||||
@@ -188,7 +204,11 @@ impl std::fmt::Debug for PumpedWebSocket {
|
||||
|
||||
impl PumpedWebSocket {
|
||||
/// 用底层流启动一个 pump 任务并返回句柄。
|
||||
pub(crate) fn new(inner: RawWsStream, keepalive: PumpKeepalive) -> Self {
|
||||
pub(crate) fn new(
|
||||
inner: RawWsStream,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
) -> Self {
|
||||
let (tx_command, rx_command) = mpsc::channel::<PumpCommand>(PUMP_COMMAND_BUFFER);
|
||||
let (tx_message, rx_message) = mpsc::channel(PUMP_MESSAGE_BUFFER);
|
||||
let connection_id = Uuid::new_v4();
|
||||
@@ -203,6 +223,7 @@ impl PumpedWebSocket {
|
||||
rx_command,
|
||||
tx_message,
|
||||
keepalive,
|
||||
context,
|
||||
closed_for_task,
|
||||
exit_reason_for_task,
|
||||
)
|
||||
@@ -273,12 +294,17 @@ impl Drop for PumpedWebSocket {
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "pump owns one flat set of channel/state handles and log context"
|
||||
)]
|
||||
async fn pump_loop(
|
||||
connection_id: Uuid,
|
||||
mut inner: RawWsStream,
|
||||
mut rx_command: mpsc::Receiver<PumpCommand>,
|
||||
tx_message: mpsc::Sender<Result<Message, tungstenite::Error>>,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
closed: Arc<AtomicBool>,
|
||||
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
|
||||
) {
|
||||
@@ -429,7 +455,7 @@ async fn pump_loop(
|
||||
*exit_reason
|
||||
.lock()
|
||||
.expect("WebSocket pump exit reason lock poisoned") = Some(reason.clone());
|
||||
log_pump_exit(connection_id, &reason, backpressure_events);
|
||||
log_pump_exit(connection_id, &context, &reason, backpressure_events);
|
||||
|
||||
if reason.should_send_close() {
|
||||
let _ = inner.send(Message::Close(None)).await;
|
||||
@@ -510,11 +536,20 @@ async fn send_keepalive_ping(
|
||||
}))
|
||||
}
|
||||
|
||||
fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_events: u64) {
|
||||
fn log_pump_exit(
|
||||
connection_id: Uuid,
|
||||
context: &PumpLogContext,
|
||||
reason: &PumpExitReason,
|
||||
backpressure_events: u64,
|
||||
) {
|
||||
let detail = reason.detail().unwrap_or_default();
|
||||
let account_id = context.account_id.as_deref().unwrap_or("unknown");
|
||||
let conversation_id_hash = context.conversation_id_hash.as_deref().unwrap_or("unknown");
|
||||
if reason.is_unexpected() {
|
||||
tracing::warn!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
@@ -523,6 +558,8 @@ fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_even
|
||||
} else if matches!(reason, PumpExitReason::UpstreamCloseFrame { .. }) {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
@@ -531,6 +568,8 @@ fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_even
|
||||
} else {
|
||||
tracing::debug!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
|
||||
@@ -15,7 +15,7 @@ use gateway_admin::model::provider_credentials::{
|
||||
AuthorizationMutationTarget, AuthorizationOwnerBinding, CompleteAuthorization,
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, ProviderDocument, ProviderExportCredentialInput,
|
||||
ProviderQuotaRequest,
|
||||
ProviderQuotaRequest, QuotaLocalUsageAttribution,
|
||||
};
|
||||
use gateway_admin::model::{MutationActor, MutationContext, Revision};
|
||||
use gateway_admin::ports::provider::ProviderAdminErrorKind;
|
||||
@@ -82,6 +82,22 @@ async fn openai_bundle_exposes_one_core_provider_and_drains_worker_contributions
|
||||
panic!("Desktop release worker must be scheduled");
|
||||
};
|
||||
assert_eq!(schedule.interval(), APPCAST_POLL_INTERVAL);
|
||||
let cli_release_worker = contributions
|
||||
.iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.owner() == "openai-cli-release" =>
|
||||
{
|
||||
Some(registration)
|
||||
}
|
||||
WorkerContribution::Registration(_) | WorkerContribution::Disabled { .. } => None,
|
||||
})
|
||||
.expect("CLI release worker");
|
||||
assert_eq!(cli_release_worker.id.kind(), WorkerKind::QuotaCatalogHealth);
|
||||
let WorkerRunnable::Scheduled { schedule, .. } = &cli_release_worker.runnable else {
|
||||
panic!("CLI release worker must be scheduled");
|
||||
};
|
||||
assert_eq!(schedule.interval(), APPCAST_POLL_INTERVAL);
|
||||
assert!(contributions.iter().any(|contribution| {
|
||||
matches!(
|
||||
contribution,
|
||||
@@ -101,13 +117,21 @@ async fn openai_admin_provider_exposes_live_wire_profile_and_validated_billing()
|
||||
.expect("OpenAI bundle");
|
||||
let admin = bundle.admin_provider();
|
||||
let profile = admin.dashboard_wire_profile().expect("wire profile");
|
||||
assert_eq!(profile.version, "0.102.0");
|
||||
assert_eq!(profile.build, None);
|
||||
assert_eq!(profile.target.os_type, "Mac OS");
|
||||
assert_eq!(profile.target.os_version, "15.5.0");
|
||||
assert_eq!(
|
||||
profile.user_agent,
|
||||
"Codex Desktop/0.102.0 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 1.2026.190)"
|
||||
);
|
||||
assert_eq!(
|
||||
profile
|
||||
.attributes
|
||||
.iter()
|
||||
.find(|attribute| attribute.label == "Codex Core")
|
||||
.find(|attribute| attribute.label == "客户端标识")
|
||||
.map(|attribute| attribute.value.as_str()),
|
||||
Some("0.102.0")
|
||||
Some("Codex Desktop; 1.2026.190")
|
||||
);
|
||||
assert_eq!(
|
||||
profile.release.as_ref().map(|release| release.status),
|
||||
@@ -428,7 +452,8 @@ async fn openai_admin_provider_projects_cached_quota_models_and_canonical_export
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_quota() {
|
||||
async fn openai_admin_provider_projects_official_codex_quota_and_independent_buckets_with_chinese_labels()
|
||||
{
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
@@ -444,11 +469,15 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
.account("acct_admin_canonical_quota")
|
||||
.expect("stored account");
|
||||
let raw = json!({
|
||||
"active_limit": "premium",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 91,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 88
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
@@ -459,6 +488,29 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
"used_percent": 2,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 0
|
||||
}
|
||||
}
|
||||
}, {
|
||||
"limit_name": "code_review",
|
||||
"metered_feature": "code_review",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 12,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}, {
|
||||
"limit_name": "GPT-5.3-Codex-Spark",
|
||||
"metered_feature": "codex_bengalfox",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 0,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}]
|
||||
@@ -471,6 +523,7 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -498,9 +551,48 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].label, "月限额");
|
||||
assert_eq!(monthly[0].source.as_deref(), Some("core"));
|
||||
assert_eq!(monthly[0].used_percent, Some(2.0));
|
||||
assert!(monthly.iter().any(|window| {
|
||||
window.label == "月额度"
|
||||
&& window.source.as_deref() == Some("codex")
|
||||
&& window.used_percent == Some(91.0)
|
||||
}));
|
||||
assert!(
|
||||
!quota
|
||||
.windows
|
||||
.iter()
|
||||
.any(|window| window.key.starts_with("additional-0-codex")),
|
||||
"the additional codex alias should not become a second display bucket"
|
||||
);
|
||||
let secondary = quota
|
||||
.windows
|
||||
.iter()
|
||||
.find(|window| window.key == "core-secondary")
|
||||
.expect("core secondary quota");
|
||||
assert_eq!(secondary.label, "次级额度");
|
||||
assert_eq!(secondary.used_percent, Some(88.0));
|
||||
let review = quota
|
||||
.windows
|
||||
.iter()
|
||||
.find(|window| window.source.as_deref() == Some("code_review"))
|
||||
.expect("code review quota");
|
||||
assert_eq!(review.label, "代码审查 · 周额度");
|
||||
let spark = quota
|
||||
.windows
|
||||
.iter()
|
||||
.find(|window| window.source.as_deref() == Some("codex_bengalfox"))
|
||||
.expect("Spark quota");
|
||||
assert_eq!(
|
||||
(
|
||||
monthly[0].local_usage_attribution,
|
||||
review.local_usage_attribution,
|
||||
spark.local_usage_attribution,
|
||||
),
|
||||
(
|
||||
QuotaLocalUsageAttribution::AccountWide,
|
||||
QuotaLocalUsageAttribution::Unavailable,
|
||||
QuotaLocalUsageAttribution::Unavailable,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -536,16 +628,16 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist raw quota");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -583,11 +675,10 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -645,7 +736,6 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit(
|
||||
stale_record.next_refresh_at = None;
|
||||
stale_record.enabled = false;
|
||||
stale_record.availability = AdminAccountAvailability::Banned;
|
||||
stale_record.cooldown_until = Some(Utc::now() + chrono::Duration::minutes(5));
|
||||
let rotation_error = admin
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: stale_record,
|
||||
@@ -702,7 +792,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
provider_kind: account.provider().clone(),
|
||||
name: account.name().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
upstream_user_id: account.upstream_user_id().to_owned(),
|
||||
upstream_user_id: account.upstream_user_id().map(str::to_owned),
|
||||
upstream_account_id: account.upstream_account_id().map(str::to_owned),
|
||||
plan_type: account.plan_type().map(str::to_owned),
|
||||
authentication_kind: account.authentication_kind().to_owned(),
|
||||
@@ -712,9 +802,8 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability: AdminAccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -733,7 +822,7 @@ fn valid_config() -> TestOpenAiConfig {
|
||||
codex_version: "0.102.0".to_owned(),
|
||||
desktop_version: "1.2026.190".to_owned(),
|
||||
desktop_build: "19012345678".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
@@ -901,6 +990,13 @@ impl ProviderCooldownPort for TestCooldown {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
struct TestRuntimePolicy;
|
||||
|
||||
@@ -13,7 +13,7 @@ fn openai_config_builds_the_audited_wire_profile() {
|
||||
|
||||
assert_eq!(
|
||||
profile.user_agent(),
|
||||
"Codex Desktop/0.102.0 (macOS 15.5.0; arm64) xterm-256color"
|
||||
"Codex Desktop/0.102.0 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 1.2026.190)"
|
||||
);
|
||||
assert_eq!(profile.desktop_build, "19012345678");
|
||||
}
|
||||
@@ -124,6 +124,10 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
"https://auth.openai.com/oauth/token",
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
config.wire_profile_state().snapshot().user_agent(),
|
||||
"Codex Desktop/0.146.0 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.727.51351)"
|
||||
);
|
||||
}
|
||||
|
||||
fn valid_config() -> OpenAiConfig {
|
||||
@@ -133,7 +137,7 @@ fn valid_config() -> OpenAiConfig {
|
||||
codex_version: "0.102.0".to_owned(),
|
||||
desktop_version: "1.2026.190".to_owned(),
|
||||
desktop_build: "19012345678".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
|
||||
@@ -1,804 +1,97 @@
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use chrono::{TimeZone, Utc};
|
||||
use ed25519_dalek::SigningKey;
|
||||
use ed25519_dalek::pkcs8::EncodePrivateKey as _;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, LoadedCredential, PlaintextCredential,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, ProviderRefreshLeaseRequest,
|
||||
ProviderStoreError,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use provider_openai::credential::token_client::{RefreshFailure, TokenPair, TokenRefresher};
|
||||
use provider_openai::credential::{
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CodexAccountIdentityVerifier,
|
||||
CodexAgentIdentityAuthMode, CodexAgentIdentityCredentialData, CodexCredentialAdmin,
|
||||
CodexCredentialAdminError, CodexCredentialAdminService, CodexCredentialCodec,
|
||||
CodexIdentityExpectation, CodexIdentityVerification, CodexIdentityVerificationError,
|
||||
ExportManagedCodexCredential, ImportCodexOAuthCredential, RotateManagedCodexCredential,
|
||||
};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use provider_openai::credential::{CodexCredentialAdminService, CodexCredentialCodec};
|
||||
use secrecy::ExposeSecret as _;
|
||||
|
||||
use crate::support::{MemoryAccountStore, codex_account, profile, runtime_policy, secret};
|
||||
use crate::support::{TestLeaseCoordinator, runtime_policy};
|
||||
|
||||
fn import(id: &str, token: &str) -> ImportCodexOAuthCredential {
|
||||
ImportCodexOAuthCredential {
|
||||
account_id: id.to_owned(),
|
||||
name: format!("name-{id}"),
|
||||
secret: secret(token),
|
||||
verified_account: profile(&format!("chatgpt-{id}")),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
struct UnusedRefresher;
|
||||
|
||||
#[async_trait]
|
||||
impl TokenRefresher for UnusedRefresher {
|
||||
async fn refresh(&self, _refresh_token: &str) -> Result<TokenPair, RefreshFailure> {
|
||||
panic!("a direct access-token import must not refresh its refresh token")
|
||||
}
|
||||
}
|
||||
|
||||
fn encoded_credential(account_id: &str, token: &str) -> PlaintextCredential {
|
||||
CodexCredentialCodec::encode_new(&secret(token), &profile(account_id), Vec::new())
|
||||
.expect("encode current credential")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_import_returns_core_account_with_plaintext_json() {
|
||||
let prepared = CodexCredentialAdmin
|
||||
.prepare_import(import("acct_import", "access-import"))
|
||||
.expect("prepare import");
|
||||
assert_eq!(prepared.account.id().as_str(), "acct_import");
|
||||
assert_eq!(prepared.account.provider().as_str(), "openai");
|
||||
assert_eq!(
|
||||
prepared
|
||||
.credential
|
||||
.expose_to_provider()
|
||||
.get("access_token")
|
||||
.and_then(serde_json::Value::as_str),
|
||||
Some("access-import")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_rotation_preserves_provider_owned_cookie_data() {
|
||||
let account = codex_account("acct_rotate");
|
||||
let mut credential = encoded_credential("chatgpt-acct_rotate", "old-access");
|
||||
credential
|
||||
.expose_to_provider()
|
||||
.get("access_token")
|
||||
.expect("access token");
|
||||
let mut data = CodexCredentialCodec::decode_complete(&credential).expect("decode data");
|
||||
data.oauth_mut().expect("OAuth data").oauth_client_id = Some("oauth-client".to_owned());
|
||||
credential = CodexCredentialCodec::encode_complete(data).expect("encode complete data");
|
||||
let prepared = CodexCredentialAdmin
|
||||
.prepare_rotation(RotateManagedCodexCredential {
|
||||
current: LoadedCredential {
|
||||
account,
|
||||
credential,
|
||||
},
|
||||
secret: secret("new-access"),
|
||||
verified_account: profile("chatgpt-acct_rotate"),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
})
|
||||
.expect("prepare rotation");
|
||||
let decoded = CodexCredentialCodec::decode(prepared.credential.credential())
|
||||
.expect("decode prepared credential");
|
||||
assert_eq!(
|
||||
decoded
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"new-access"
|
||||
);
|
||||
assert_eq!(decoded.oauth_client_id.as_deref(), Some("oauth-client"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_rotation_rejects_account_rebinding() {
|
||||
let error = CodexCredentialAdmin
|
||||
.prepare_rotation(RotateManagedCodexCredential {
|
||||
current: LoadedCredential {
|
||||
account: codex_account("acct_rotate"),
|
||||
credential: encoded_credential("chatgpt-acct_rotate", "old-access"),
|
||||
},
|
||||
secret: secret("new-access"),
|
||||
verified_account: profile("chatgpt-other"),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
})
|
||||
.expect_err("identity rebinding must fail");
|
||||
assert_eq!(error, CodexCredentialAdminError::IdentityMismatch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_rotation_rejects_principal_rebinding_without_full_authorization() {
|
||||
let mut rebound = profile("chatgpt-acct_rotate");
|
||||
rebound.oauth_subject = "different-subject".to_owned();
|
||||
rebound.poid = None;
|
||||
let error = CodexCredentialAdmin
|
||||
.prepare_rotation(RotateManagedCodexCredential {
|
||||
current: LoadedCredential {
|
||||
account: codex_account("acct_rotate"),
|
||||
credential: encoded_credential("chatgpt-acct_rotate", "old-access"),
|
||||
},
|
||||
secret: secret("new-access"),
|
||||
verified_account: rebound,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
})
|
||||
.expect_err("non-authorization rotation cannot replace the principal");
|
||||
|
||||
assert_eq!(error, CodexCredentialAdminError::IdentityMismatch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepared_commands_debug_never_prints_tokens() {
|
||||
let prepared = CodexCredentialAdmin
|
||||
.prepare_import(import("acct_debug", "debug-secret"))
|
||||
.expect("prepare import");
|
||||
assert!(!format!("{prepared:?}").contains("debug-secret"));
|
||||
let _: PlaintextCredential = prepared.credential;
|
||||
}
|
||||
|
||||
fn export_item(id: &str, token: &str) -> ExportManagedCodexCredential {
|
||||
ExportManagedCodexCredential {
|
||||
current: LoadedCredential {
|
||||
account: codex_account(id),
|
||||
credential: encoded_credential(&format!("chatgpt-{id}"), token),
|
||||
},
|
||||
added_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 2, 47, 1)
|
||||
.single()
|
||||
.expect("added at"),
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 19, 3, 0, 0)
|
||||
.single()
|
||||
.expect("updated at"),
|
||||
}
|
||||
}
|
||||
|
||||
fn agent_export_item(id: &str) -> (ExportManagedCodexCredential, String) {
|
||||
let signing_key = SigningKey::from_bytes(&[7; 32]);
|
||||
let private_key = STANDARD.encode(
|
||||
signing_key
|
||||
.to_pkcs8_der()
|
||||
.expect("encode Agent Identity key")
|
||||
.as_bytes(),
|
||||
);
|
||||
let credential =
|
||||
CodexCredentialCodec::encode_agent_identity(CodexAgentIdentityCredentialData {
|
||||
schema_version: 1,
|
||||
auth_mode: CodexAgentIdentityAuthMode::AgentIdentity,
|
||||
installation_id: "00000000-0000-4000-8000-000000000007".to_owned(),
|
||||
agent_runtime_id: "runtime-export".to_owned(),
|
||||
agent_private_key: private_key.clone(),
|
||||
task_id: Some("task-export".to_owned()),
|
||||
cookies: Vec::new(),
|
||||
})
|
||||
.expect("encode Agent Identity credential");
|
||||
let account = ProviderAccount::new(
|
||||
ProviderAccountId::new(id.to_owned()).expect("account id"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
format!("Agent Identity {id}"),
|
||||
"agent-user-export".to_owned(),
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY.to_owned(),
|
||||
CredentialRevision::new(1).expect("revision"),
|
||||
None,
|
||||
)
|
||||
.with_profile(
|
||||
Some("agent-export@example.com".to_owned()),
|
||||
Some("agent-account-export".to_owned()),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_refresh_schedule(false, None);
|
||||
(
|
||||
ExportManagedCodexCredential {
|
||||
current: LoadedCredential {
|
||||
account,
|
||||
credential,
|
||||
},
|
||||
added_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 2, 47, 1)
|
||||
.single()
|
||||
.expect("added at"),
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 19, 3, 0, 0)
|
||||
.single()
|
||||
.expect("updated at"),
|
||||
},
|
||||
private_key,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cpr_export_matches_the_canonical_real_document_shape() {
|
||||
let document = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![export_item("acct_export", "export-secret")])
|
||||
.expect("format export");
|
||||
assert_eq!(document.len(), 1);
|
||||
let value = document.into_json().expect("serialize export");
|
||||
assert_eq!(value["sourceFormat"], "cpr");
|
||||
let account = &value["accounts"][0];
|
||||
let mut keys = account
|
||||
.as_object()
|
||||
.expect("account object")
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>();
|
||||
keys.sort_unstable();
|
||||
assert_eq!(
|
||||
keys,
|
||||
[
|
||||
"accessTokenExpiresAt",
|
||||
"accountId",
|
||||
"addedAt",
|
||||
"email",
|
||||
"id",
|
||||
"label",
|
||||
"planType",
|
||||
"refreshToken",
|
||||
"status",
|
||||
"token",
|
||||
"updatedAt",
|
||||
"userId",
|
||||
]
|
||||
);
|
||||
assert_eq!(account["id"], "acct_export");
|
||||
assert_eq!(account["token"], "export-secret");
|
||||
assert_eq!(account["refreshToken"], "rt-export-secret");
|
||||
assert_eq!(account["status"], "active");
|
||||
assert!(
|
||||
account["addedAt"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.ends_with("+08:00"))
|
||||
);
|
||||
assert!(
|
||||
account["accessTokenExpiresAt"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.ends_with("+00:00"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cpr_export_batch_validation_and_debug_are_secret_safe() {
|
||||
assert_eq!(
|
||||
CodexCredentialAdmin
|
||||
.format_cpr_export(Vec::new())
|
||||
.expect_err("empty export"),
|
||||
CodexCredentialAdminError::InvalidInput
|
||||
);
|
||||
let duplicate = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![
|
||||
export_item("acct_duplicate_export", "first-secret"),
|
||||
export_item("acct_duplicate_export", "second-secret"),
|
||||
])
|
||||
.expect_err("duplicate export account");
|
||||
assert_eq!(duplicate, CodexCredentialAdminError::InvalidInput);
|
||||
|
||||
let document = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![export_item("acct_debug_export", "never-print-me")])
|
||||
.expect("export document");
|
||||
let debug = format!("{document:?}");
|
||||
assert!(!debug.contains("never-print-me"));
|
||||
assert!(!debug.contains("rt-never-print-me"));
|
||||
assert!(debug.contains("account_count: 1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cpr_export_preserves_agent_identity_material_without_oauth_fields() {
|
||||
let (item, private_key) = agent_export_item("acct_agent_export");
|
||||
let value = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![item])
|
||||
.expect("Agent Identity export")
|
||||
.into_json()
|
||||
.expect("serialize Agent Identity export");
|
||||
let account = &value["accounts"][0];
|
||||
|
||||
assert_eq!(account["authMode"], "agentIdentity");
|
||||
assert_eq!(account["agentRuntimeId"], "runtime-export");
|
||||
assert!(
|
||||
account["agentPrivateKey"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value == private_key)
|
||||
);
|
||||
assert_eq!(account["taskId"], "task-export");
|
||||
assert!(account.get("token").is_none());
|
||||
assert!(account.get("refreshToken").is_none());
|
||||
assert!(account.get("accessTokenExpiresAt").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cpr_export_and_import_support_mixed_oauth_and_agent_identity_accounts() {
|
||||
let (agent, _) = agent_export_item("acct_mixed_agent");
|
||||
let value = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![
|
||||
export_item("acct_mixed_oauth", "token-acct_mixed_oauth"),
|
||||
agent,
|
||||
])
|
||||
.expect("mixed export")
|
||||
.into_json()
|
||||
.expect("serialize mixed export");
|
||||
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
.prepare_import_document(value)
|
||||
.await
|
||||
.expect("mixed CPR import");
|
||||
assert_eq!(prepared.accounts().len(), 2);
|
||||
assert!(prepared.accounts().iter().any(|account| {
|
||||
account.account.authentication_kind() == CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY
|
||||
}));
|
||||
assert!(
|
||||
prepared
|
||||
.accounts()
|
||||
.iter()
|
||||
.any(|account| { account.account.authentication_kind() == "oauth" })
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) struct ManualRefresher {
|
||||
outcome: Mutex<Option<Result<TokenPair, RefreshFailure>>>,
|
||||
seen: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TokenRefresher for ManualRefresher {
|
||||
async fn refresh(&self, refresh_token: &str) -> Result<TokenPair, RefreshFailure> {
|
||||
self.seen
|
||||
.lock()
|
||||
.expect("seen refresh tokens")
|
||||
.push(refresh_token.to_owned());
|
||||
self.outcome
|
||||
.lock()
|
||||
.expect("refresh outcome")
|
||||
.take()
|
||||
.expect("one refresh outcome")
|
||||
}
|
||||
}
|
||||
|
||||
struct ManualVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for ManualVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
if secret.access_token.expose_secret() != "refreshed-access" {
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(CodexIdentityVerification::Complete(profile(
|
||||
"chatgpt-acct_manual_refresh",
|
||||
)))
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_id_token: &SecretString,
|
||||
_expected_nonce: &SecretString,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
struct DropCountGuard(Arc<AtomicUsize>);
|
||||
|
||||
impl Drop for DropCountGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
struct ManualLeases {
|
||||
requests: Mutex<Vec<ProviderRefreshLeaseRequest>>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
available: bool,
|
||||
}
|
||||
|
||||
impl ProviderLeasePort for ManualLeases {
|
||||
fn load_state<'a>(
|
||||
&'a self,
|
||||
_: &'a gateway_core::routing::ProviderKind,
|
||||
_: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<
|
||||
'a,
|
||||
Result<gateway_core::provider_ports::ProviderSchedulingState, ProviderStoreError>,
|
||||
> {
|
||||
Box::pin(async {
|
||||
Ok(gateway_core::provider_ports::ProviderSchedulingState::new(
|
||||
Default::default(),
|
||||
0,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn try_acquire(
|
||||
&self,
|
||||
request: ProviderLeaseRequest,
|
||||
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
match request {
|
||||
ProviderLeaseRequest::RefreshCapacity(_) => {}
|
||||
ProviderLeaseRequest::Refresh(request) => {
|
||||
self.requests.lock().expect("lease requests").push(request);
|
||||
}
|
||||
ProviderLeaseRequest::Scheduling(_) => panic!("unexpected scheduling lease"),
|
||||
}
|
||||
Ok(if self.available {
|
||||
ProviderLeaseAcquisition::Acquired(Box::new(DropCountGuard(Arc::clone(
|
||||
&self.drops,
|
||||
))))
|
||||
} else {
|
||||
ProviderLeaseAcquisition::Busy { retry_after: None }
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn manual_refresh_fixture(
|
||||
outcome: Result<TokenPair, RefreshFailure>,
|
||||
available: bool,
|
||||
) -> (
|
||||
Arc<MemoryAccountStore>,
|
||||
Arc<ManualRefresher>,
|
||||
Arc<ManualLeases>,
|
||||
CodexCredentialAdminService,
|
||||
) {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_manual_refresh".to_owned(),
|
||||
name: "manual refresh".to_owned(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: profile("chatgpt-acct_manual_refresh"),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let refresher = Arc::new(ManualRefresher {
|
||||
outcome: Mutex::new(Some(outcome)),
|
||||
seen: Mutex::new(Vec::new()),
|
||||
});
|
||||
let leases = Arc::new(ManualLeases {
|
||||
requests: Mutex::new(Vec::new()),
|
||||
drops: Arc::new(AtomicUsize::new(0)),
|
||||
available,
|
||||
});
|
||||
async fn direct_import_persists_opaque_tokens_without_profile_or_token_validation() {
|
||||
let service = CodexCredentialAdminService::new(
|
||||
refresher.clone(),
|
||||
Arc::new(ManualVerifier),
|
||||
leases.clone(),
|
||||
Arc::new(UnusedRefresher),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
runtime_policy(),
|
||||
);
|
||||
(store, refresher, leases, service)
|
||||
}
|
||||
|
||||
fn refreshed_tokens() -> TokenPair {
|
||||
TokenPair {
|
||||
access_token: "refreshed-access".to_owned(),
|
||||
refresh_token: Some("rotated-refresh".to_owned()),
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_refresh_prepares_revision_fenced_rotation_without_store_mutation() {
|
||||
let (store, refresher, leases, service) =
|
||||
manual_refresh_fixture(Ok(refreshed_tokens()), true).await;
|
||||
let account_id = ProviderAccountId::new("acct_manual_refresh").expect("account id");
|
||||
let current = store
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.expect("current credential");
|
||||
let prepared = service
|
||||
.manual_refresh(current)
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"token": " ",
|
||||
"refresh_token": "",
|
||||
"id_token": "not.a.parseable.jwt"
|
||||
}))
|
||||
.await
|
||||
.expect("manual refresh");
|
||||
.expect("opaque direct import must be accepted");
|
||||
|
||||
assert_eq!(prepared.credential.account_id(), &account_id);
|
||||
assert_eq!(prepared.credential.expected_revision().get(), 1);
|
||||
let runtime = CodexCredentialCodec::decode(prepared.credential.credential())
|
||||
.expect("prepared credential");
|
||||
let account = prepared.accounts().first().expect("one prepared account");
|
||||
let runtime = CodexCredentialCodec::decode(&account.credential).expect("stored credential");
|
||||
let oauth = runtime.authentication.oauth().expect("OAuth credential");
|
||||
assert_eq!(oauth.access_token.expose_secret(), " ");
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"refreshed-access"
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
oauth
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.expect("rotated refresh")
|
||||
.expect("provided refresh token")
|
||||
.expose_secret(),
|
||||
"rotated-refresh"
|
||||
""
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_manual_refresh")
|
||||
.expect("account")
|
||||
.revision()
|
||||
.get(),
|
||||
1
|
||||
oauth
|
||||
.id_token
|
||||
.as_ref()
|
||||
.expect("provided ID token")
|
||||
.expose_secret(),
|
||||
"not.a.parseable.jwt"
|
||||
);
|
||||
assert_eq!(
|
||||
refresher.seen.lock().expect("seen tokens").as_slice(),
|
||||
["rt-old-access"]
|
||||
);
|
||||
assert_eq!(leases.drops.load(Ordering::SeqCst), 0);
|
||||
drop(prepared);
|
||||
assert_eq!(leases.drops.load(Ordering::SeqCst), 2);
|
||||
assert!(account.account.upstream_user_id().is_none());
|
||||
assert!(account.account.access_token_expires_at().is_none());
|
||||
assert!(account.account.next_refresh_at().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_refresh_missing_lease_fails_before_exchange() {
|
||||
let (store, unavailable_refresher, _, unavailable_service) =
|
||||
manual_refresh_fixture(Ok(refreshed_tokens()), false).await;
|
||||
let account_id = ProviderAccountId::new("acct_manual_refresh").expect("account id");
|
||||
let current = store
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.expect("current credential");
|
||||
let unavailable = unavailable_service
|
||||
.manual_refresh(current)
|
||||
.await
|
||||
.expect_err("lease unavailable");
|
||||
assert_eq!(
|
||||
unavailable,
|
||||
CodexCredentialAdminError::RefreshLeaseUnavailable
|
||||
async fn direct_import_projects_access_token_jwt_expiry_without_persisting_refresh_margin() {
|
||||
let expires_at = Utc
|
||||
.timestamp_opt(2_000_000_000, 0)
|
||||
.single()
|
||||
.expect("valid test timestamp");
|
||||
let access_token = format!(
|
||||
"unverified-header.{}.unverified-signature",
|
||||
URL_SAFE_NO_PAD.encode(
|
||||
serde_json::to_vec(&serde_json::json!({"exp": expires_at.timestamp()}))
|
||||
.expect("test JWT payload"),
|
||||
)
|
||||
);
|
||||
assert!(
|
||||
unavailable_refresher
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen tokens")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
struct ImportVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for ImportVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
let suffix = secret
|
||||
.access_token
|
||||
.expose_secret()
|
||||
.strip_prefix("token-")
|
||||
.ok_or(CodexIdentityVerificationError::Rejected)?;
|
||||
let mut profile = profile(&format!("chatgpt-{suffix}"));
|
||||
if let Some(user) = suffix.strip_prefix("shared-") {
|
||||
profile.chatgpt_account_id = "chatgpt-shared".to_owned();
|
||||
profile.chatgpt_user_id = format!("user-{user}");
|
||||
}
|
||||
if expectation
|
||||
.chatgpt_account_id()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_account_id)
|
||||
|| expectation
|
||||
.chatgpt_user_id()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_user_id)
|
||||
{
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(CodexIdentityVerification::Complete(profile))
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_id_token: &SecretString,
|
||||
_expected_nonce: &SecretString,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn import_service(
|
||||
refresher: Arc<ManualRefresher>,
|
||||
) -> provider_openai::credential::CodexCredentialAdminService {
|
||||
CodexCredentialAdminService::new(
|
||||
refresher,
|
||||
Arc::new(ImportVerifier),
|
||||
Arc::new(ManualLeases {
|
||||
requests: Mutex::new(Vec::new()),
|
||||
drops: Arc::new(AtomicUsize::new(0)),
|
||||
available: true,
|
||||
}),
|
||||
let service = CodexCredentialAdminService::new(
|
||||
Arc::new(UnusedRefresher),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
runtime_policy(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn unused_import_refresher() -> Arc<ManualRefresher> {
|
||||
Arc::new(ManualRefresher {
|
||||
outcome: Mutex::new(Some(Err(RefreshFailure::InvalidGrant))),
|
||||
seen: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_import_uses_only_tokens_and_verified_identity() {
|
||||
let refresher = unused_import_refresher();
|
||||
let prepared = import_service(refresher.clone())
|
||||
);
|
||||
let prepared = service
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_cpr_import",
|
||||
"email": "cpr@example.com",
|
||||
"accountId": "chatgpt-cpr",
|
||||
"userId": "user-chatgpt-cpr",
|
||||
"label": "CPR account",
|
||||
"planType": "pro",
|
||||
"token": "Bearer token-cpr",
|
||||
"refreshToken": "refresh-cpr",
|
||||
"accessTokenExpiresAt": "2100-01-01T00:00:00+00:00",
|
||||
"status": "disabled",
|
||||
"addedAt": "2026-07-18T10:47:01+08:00",
|
||||
"updatedAt": "2026-07-19T11:00:00+08:00",
|
||||
"unrelated": { "value": true }
|
||||
}]
|
||||
"token": access_token,
|
||||
"refresh_token": "refresh-token"
|
||||
}))
|
||||
.await
|
||||
.expect("CPR import");
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
let account = &prepared.accounts()[0];
|
||||
assert!(account.account.id().as_str().starts_with("acct_"));
|
||||
assert!(account.account.enabled());
|
||||
assert_eq!(account.account.upstream_account_id(), Some("chatgpt-cpr"));
|
||||
let runtime = CodexCredentialCodec::decode(&account.credential).expect("credential");
|
||||
.expect("direct JWT import must be accepted");
|
||||
|
||||
let account = prepared.accounts().first().expect("one prepared account");
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"token-cpr"
|
||||
account
|
||||
.account
|
||||
.access_token_expires_at()
|
||||
.map(chrono::DateTime::<Utc>::from),
|
||||
Some(expires_at)
|
||||
);
|
||||
assert!(refresher.seen.lock().expect("seen tokens").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cpr_batch_allows_distinct_users_in_the_same_workspace() {
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_shared_a",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
}, {
|
||||
"id": "acct_shared_b",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-b",
|
||||
"token": "token-shared-b"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect("distinct users sharing one workspace are separate credentials");
|
||||
|
||||
assert_eq!(prepared.accounts().len(), 2);
|
||||
assert!(
|
||||
prepared
|
||||
.accounts()
|
||||
.iter()
|
||||
.all(|account| { account.account.upstream_account_id() == Some("chatgpt-shared") })
|
||||
);
|
||||
|
||||
let duplicate = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_duplicate_a",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
}, {
|
||||
"id": "acct_duplicate_b",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect_err("the same upstream user and workspace must stay unique");
|
||||
assert_eq!(duplicate, CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_import_extracts_tokens_without_parsing_source_projections() {
|
||||
let bundle = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"exported_at": "2026-07-03T15:46:38.717Z",
|
||||
"proxies": [],
|
||||
"accounts": [{
|
||||
"name": "bundle@example.com",
|
||||
"platform": "openai",
|
||||
"type": "oauth",
|
||||
"credentials": {
|
||||
"access_token": "token-bundle",
|
||||
"refresh_token": "refresh-bundle",
|
||||
"chatgpt_account_id": "chatgpt-bundle",
|
||||
"chatgpt_user_id": "user-chatgpt-bundle"
|
||||
},
|
||||
"concurrency": 3,
|
||||
"priority": 50
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect("credential bundle import");
|
||||
let auth_document = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"accounts": [{
|
||||
"type": "openai",
|
||||
"access_token": "token-auth-document",
|
||||
"refresh_token": "refresh-auth-document",
|
||||
"chatgpt_account_id": "chatgpt-auth-document",
|
||||
"chatgpt_user_id": "user-chatgpt-auth-document",
|
||||
"email": "auth-document@example.com",
|
||||
"label": "Auth document"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect("auth document import");
|
||||
|
||||
for (prepared, expected) in [
|
||||
(&bundle, "chatgpt-bundle"),
|
||||
(&auth_document, "chatgpt-auth-document"),
|
||||
] {
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
assert_eq!(
|
||||
prepared.accounts()[0].account.upstream_account_id(),
|
||||
Some(expected)
|
||||
);
|
||||
assert!(
|
||||
prepared.accounts()[0]
|
||||
.account
|
||||
.id()
|
||||
.as_str()
|
||||
.starts_with("acct_")
|
||||
);
|
||||
}
|
||||
assert!(!format!("{bundle:?} {auth_document:?}").contains("refresh-bundle"));
|
||||
assert!(!format!("{bundle:?} {auth_document:?}").contains("refresh-auth-document"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cliproxyapi_codex_auth_file_is_recognized_as_an_openai_auth_document() {
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"type": "codex",
|
||||
"access_token": "token-cpa",
|
||||
"refresh_token": "refresh-cpa",
|
||||
"id_token": "id-cpa",
|
||||
"account_id": "chatgpt-cpa",
|
||||
"email": "cpa@example.com",
|
||||
"expired": "2100-01-01T00:00:00Z"
|
||||
}))
|
||||
.await
|
||||
.expect("CLIProxyAPI Codex auth file import");
|
||||
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
let account = &prepared.accounts()[0].account;
|
||||
assert_eq!(account.upstream_account_id(), Some("chatgpt-cpa"));
|
||||
assert_eq!(account.email(), Some("chatgpt-cpa@example.com"));
|
||||
assert_eq!(account.authentication_kind(), "oauth");
|
||||
assert!(account.account.next_refresh_at().is_none());
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ use reqwest::StatusCode;
|
||||
use secrecy::ExposeSecret as _;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::admin::{import_service, unused_import_refresher};
|
||||
use crate::support::MemoryAccountStore;
|
||||
|
||||
fn signing_fixture(seed: u8) -> (SigningKey, String) {
|
||||
@@ -56,7 +55,7 @@ fn agent_account(
|
||||
account_id,
|
||||
provider,
|
||||
"Agent Identity test".to_owned(),
|
||||
"agent-user".to_owned(),
|
||||
Some("agent-user".to_owned()),
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY.to_owned(),
|
||||
CredentialRevision::new(1).expect("revision"),
|
||||
None,
|
||||
@@ -66,7 +65,7 @@ fn agent_account(
|
||||
Some("agent-account".to_owned()),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(false, None);
|
||||
NewProviderAccount {
|
||||
account,
|
||||
@@ -232,29 +231,3 @@ async fn task_registration_and_recovery_are_revision_fenced() {
|
||||
assert_eq!(fenced.account.revision().get(), 3);
|
||||
assert_eq!(registrar.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn agent_identity_import_accepts_the_minimal_document_shape() {
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"auth_mode": "agentIdentity",
|
||||
"agent_identity": {
|
||||
"agent_runtime_id": "runtime-minimal",
|
||||
"agent_private_key": signing_fixture(9).1,
|
||||
"account_id": "chatgpt-minimal",
|
||||
"chatgpt_user_id": "user-minimal",
|
||||
"email": "minimal@example.com",
|
||||
"plan_type": "pro"
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.expect("minimal Agent Identity import");
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
let account = &prepared.accounts()[0].account;
|
||||
assert_eq!(
|
||||
account.authentication_kind(),
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY
|
||||
);
|
||||
assert_eq!(account.access_token_expires_at(), None);
|
||||
assert!(!format!("{prepared:?}").contains("agent_private_key"));
|
||||
}
|
||||
|
||||
@@ -4,15 +4,21 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use futures::executor::block_on;
|
||||
use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
|
||||
AccountStateChange, ProviderAccountId, ProviderAccountStore as _, RotationStrategy,
|
||||
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
|
||||
QuotaObservation, QuotaWriteOutcome, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAffinityPort};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::OFFICIAL_CODEX_BASE_URL;
|
||||
use provider_openai::credential::{
|
||||
@@ -22,11 +28,12 @@ use provider_openai::credential::{
|
||||
};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::json;
|
||||
use url::Url;
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
};
|
||||
|
||||
fn create_account(store: &Arc<MemoryAccountStore>, id: &str, token: &str) {
|
||||
@@ -99,6 +106,7 @@ fn selector_with_affinity(
|
||||
session_affinity,
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,6 +116,7 @@ fn selector_with_runtime(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> CodexCredentialSelector {
|
||||
let profile = CodexWireProfileState::new(CodexWireProfile {
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -136,6 +145,7 @@ fn selector_with_runtime(
|
||||
http,
|
||||
OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
cooldowns,
|
||||
));
|
||||
CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
@@ -224,7 +234,7 @@ fn codec_reimport_preserves_existing_installation_id_for_the_same_principal() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_reimport_rejects_installation_reuse_across_principals() {
|
||||
fn codec_reimport_preserves_installation_id_without_principal_validation() {
|
||||
let existing = CodexCredentialCodec::encode_new(
|
||||
&secret("existing-access-token"),
|
||||
&profile("chatgpt-existing-principal"),
|
||||
@@ -238,7 +248,19 @@ fn codec_reimport_rejects_installation_reuse_across_principals() {
|
||||
)
|
||||
.expect("incoming credential");
|
||||
|
||||
assert!(CodexCredentialCodec::preserve_installation_id(&incoming, &existing).is_err());
|
||||
let existing_id = CodexCredentialCodec::decode_complete(&existing)
|
||||
.expect("existing data")
|
||||
.installation_id()
|
||||
.to_owned();
|
||||
let preserved = CodexCredentialCodec::preserve_installation_id(&incoming, &existing)
|
||||
.expect("preserve installation ID");
|
||||
let preserved = CodexCredentialCodec::decode_complete(&preserved).expect("preserved data");
|
||||
|
||||
assert_eq!(preserved.installation_id(), existing_id);
|
||||
assert_eq!(
|
||||
preserved.oauth().expect("OAuth data").access_token,
|
||||
"incoming-access-token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -464,6 +486,7 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
|
||||
affinity,
|
||||
Arc::clone(&account_feedback),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
for _ in 0..4 {
|
||||
account_feedback.report(
|
||||
@@ -507,11 +530,10 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -550,57 +572,6 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selector_should_escape_an_affinity_account_during_cooldown() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_first", "at-first");
|
||||
create_account(&store, "acct_second", "at-second");
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: Some(SystemTime::now() + Duration::from_secs(60)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark first account cooling down");
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let key = ProviderSessionAffinityKey::try_new("cooldown-session").expect("affinity key");
|
||||
affinity
|
||||
.bind(&provider, &key, first.id(), Duration::from_secs(60))
|
||||
.await
|
||||
.expect("seed affinity");
|
||||
let selector =
|
||||
selector_with_affinity(&store, Arc::new(TestLeaseCoordinator::default()), affinity);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
let request_attempt = attempt(BTreeSet::new());
|
||||
|
||||
let selected = selector
|
||||
.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &request_url,
|
||||
attempt: &request_attempt,
|
||||
session_affinity_key: Some(&key),
|
||||
})
|
||||
.await
|
||||
.expect("select fallback account");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
selected.account_id().as_str(),
|
||||
selected.affinity_hit(),
|
||||
selected.escape_reason(),
|
||||
selected.account_switch(),
|
||||
),
|
||||
("acct_second", false, Some("cooldown"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_honors_attempt_local_account_exclusion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -716,8 +687,12 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
}),
|
||||
)
|
||||
.expect("select account");
|
||||
block_on(selector.record_failure(lease.account(), CodexAccountFailure::CredentialExpired))
|
||||
.expect("record credential expiry");
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CredentialExpired,
|
||||
None,
|
||||
))
|
||||
.expect("record credential expiry");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
@@ -725,13 +700,58 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
.availability(),
|
||||
AccountAvailability::Expired
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message("acct_primary").as_deref(),
|
||||
Some("upstream_credential_expired")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() {
|
||||
fn credential_expired_failure_keeps_expired_oauth_for_bounded_refresh_recovery() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let mut expired_profile = profile("chatgpt-acct_primary");
|
||||
expired_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() - chrono::Duration::minutes(1));
|
||||
block_on(store.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_primary".to_owned(),
|
||||
name: "acct_primary".to_owned(),
|
||||
secret: secret("at-primary"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(10)),
|
||||
enabled: true,
|
||||
}));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::CredentialExpired,
|
||||
Some("token_expired".to_owned()),
|
||||
))
|
||||
.expect("record credential expiry");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account retained for refresh")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
let attempt = attempt(BTreeSet::new());
|
||||
let lease =
|
||||
block_on(
|
||||
@@ -750,15 +770,22 @@ fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
|
||||
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
store.quota_json("acct_primary").is_none(),
|
||||
"429 must not synthesize quota window"
|
||||
);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown expiring in the future"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -775,17 +802,13 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("record usage-limit exhaustion");
|
||||
|
||||
let account = store.account("acct_primary").expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
}
|
||||
|
||||
@@ -795,11 +818,10 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
@@ -810,9 +832,11 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability,402 的 QuotaExhausted 保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
@@ -820,47 +844,6 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_keeps_longer_existing_quota_deadline() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let existing_deadline = SystemTime::now() + Duration::from_secs(300);
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(existing_deadline),
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
))
|
||||
.expect("record shorter rate-limit failure");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until(),
|
||||
Some(existing_deadline)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -868,13 +851,22 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
@@ -883,22 +875,27 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_rate_limit_recovers_stale_authentication_state() {
|
||||
fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Invalid,
|
||||
reason: Some("identity_verification_required".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark account invalid");
|
||||
@@ -910,15 +907,17 @@ fn authenticated_rate_limit_recovers_stale_authentication_state() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability(限流不改变账号可用性),认证终态保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
@@ -933,11 +932,10 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: stale,
|
||||
reason: Some("stale_observation".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("seed stale state");
|
||||
@@ -973,12 +971,14 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
selector.record_failure(
|
||||
&second,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
);
|
||||
first_result.expect("record first account failure");
|
||||
@@ -996,7 +996,7 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
.expect("persisted second account")
|
||||
.availability(),
|
||||
],
|
||||
[AccountAvailability::QuotaExhausted; 2]
|
||||
[AccountAvailability::Ready; 2]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1006,7 +1006,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
@@ -1015,6 +1015,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
@@ -1038,6 +1039,131 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordinator() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_original", "at-original");
|
||||
create_account(&store, "acct_fallback", "at-fallback");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let original = store.account("acct_original").expect("original account");
|
||||
block_on(strict_selector.record_failure(&original, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark original account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.id().clone(),
|
||||
);
|
||||
let attempt = AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_native_continuation").expect("request id"),
|
||||
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
|
||||
),
|
||||
NonZeroU32::new(1).expect("attempt"),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, None),
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
let error =
|
||||
block_on(
|
||||
selector.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
|
||||
.expect("request URL"),
|
||||
attempt: &attempt,
|
||||
session_affinity_key: None,
|
||||
}),
|
||||
)
|
||||
.expect_err("the selector must surface the unavailable native account");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CredentialSelectionError::NoEligibleCredential
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordinator() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_original_signal", "at-original-signal");
|
||||
create_account(&store, "acct_fallback_signal", "at-fallback-signal");
|
||||
let original = store
|
||||
.account("acct_original_signal")
|
||||
.expect("original account");
|
||||
let quota = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 98}
|
||||
}
|
||||
});
|
||||
let outcome = block_on(store.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: original.id().clone(),
|
||||
expected_revision: original.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
}))
|
||||
.expect("persist quota signal");
|
||||
assert!(matches!(outcome, QuotaWriteOutcome::Updated));
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.id().clone(),
|
||||
);
|
||||
let attempt = AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_native_continuation_signal").expect("request id"),
|
||||
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
|
||||
),
|
||||
NonZeroU32::new(1).expect("attempt"),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, None),
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
let error =
|
||||
block_on(
|
||||
selector.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
|
||||
.expect("request URL"),
|
||||
attempt: &attempt,
|
||||
session_affinity_key: None,
|
||||
}),
|
||||
)
|
||||
.expect_err("the selector must surface the quota-limited native account");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CredentialSelectionError::NoEligibleCredential
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_verification_failure_isolates_only_selected_account() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1060,6 +1186,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::IdentityVerificationRequired,
|
||||
None,
|
||||
))
|
||||
.expect("record identity verification failure");
|
||||
|
||||
@@ -1082,7 +1209,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloudflare_challenge_backoff_escalates_and_success_resets_it() {
|
||||
fn cloudflare_challenge_does_not_change_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
@@ -1099,48 +1226,22 @@ fn cloudflare_challenge_backoff_escalates_and_success_resets_it() {
|
||||
)
|
||||
.expect("select account");
|
||||
|
||||
// Cloudflare 挑战只走内存退避表,不写 availability(限流不改变账号可用性)。
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record first challenge");
|
||||
let first = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("first cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(first >= Duration::from_secs(9) && first <= Duration::from_secs(10));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
))
|
||||
.expect("record second challenge");
|
||||
let second = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("second cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(second >= Duration::from_secs(29) && second <= Duration::from_secs(30));
|
||||
.expect("record challenge");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
block_on(selector.record_success(lease.account(), None));
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
))
|
||||
.expect("record reset challenge");
|
||||
let reset = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("reset cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(reset >= Duration::from_secs(9) && reset <= Duration::from_secs(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1166,9 +1267,11 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
|
||||
.expect("select account");
|
||||
|
||||
for _ in 0..3 {
|
||||
block_on(
|
||||
selector.record_failure(lease.account(), CodexAccountFailure::CloudflarePathBlocked),
|
||||
)
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
}
|
||||
|
||||
@@ -1222,18 +1325,17 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() {
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record challenge");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let cooldown_until = account.cooldown_until().expect("cooldown");
|
||||
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
|
||||
assert_eq!(data.cookies().len(), 1);
|
||||
assert!(
|
||||
data.cookies()[0]
|
||||
.expires_at
|
||||
.is_some_and(|expires_at| SystemTime::from(expires_at) <= cooldown_until)
|
||||
);
|
||||
assert!(data.cookies()[0].expires_at.is_some_and(|expires_at| {
|
||||
let expires_at = SystemTime::from(expires_at);
|
||||
expires_at > SystemTime::now() && expires_at <= SystemTime::now() + Duration::from_secs(120)
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1267,8 +1369,12 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
|
||||
session_affinity_key: None,
|
||||
}))
|
||||
.expect("select revised account");
|
||||
block_on(selector.record_failure(second.account(), CodexAccountFailure::CloudflarePathBlocked))
|
||||
.expect("record path block");
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
|
||||
@@ -1304,7 +1410,7 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
|
||||
|
||||
assert_eq!(outcome.credential_revision, Some(current.revision().get()));
|
||||
assert_ne!(current.revision(), lease.account().revision());
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("record failure with current revision");
|
||||
assert_eq!(
|
||||
store
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user