mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-08 16:05:33 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab | ||
|
|
95f6434443 | ||
|
|
8734b0d3d5 | ||
|
|
790d0d9628 | ||
|
|
3c562563c5 | ||
|
|
f19785d0e1 | ||
|
|
a3d098a87c | ||
|
|
449d7c4218 | ||
|
|
d199141a91 | ||
|
|
df83ffdafb | ||
|
|
cf03c56cfa | ||
|
|
3d1e33558a | ||
|
|
f314ebb3a2 | ||
|
|
31d6ef9585 | ||
|
|
fdfb77c4a2 | ||
|
|
a93e3853f7 |
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -15,7 +12,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
@@ -43,14 +40,14 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
ref: ${{ inputs.tag }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Read release metadata
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
RELEASE_TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
ruby <<'RUBY' >> "$GITHUB_OUTPUT"
|
||||
require "json"
|
||||
@@ -188,9 +185,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -253,9 +250,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -378,7 +375,7 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
@@ -454,7 +451,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -471,8 +468,8 @@ jobs:
|
||||
push: true
|
||||
build-args: |
|
||||
CPR_PREBUILT_DIR=dist/docker
|
||||
cache-from: type=gha,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
labels: |
|
||||
@@ -525,7 +522,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -584,7 +581,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
|
||||
@@ -182,9 +182,10 @@ upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
|
||||
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
|
||||
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
|
||||
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
|
||||
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
|
||||
仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
Generated
+115
-101
@@ -239,7 +239,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http-body 1.0.1",
|
||||
@@ -265,7 +265,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"percent-encoding",
|
||||
@@ -566,9 +566,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "base16ct"
|
||||
version = "0.2.0"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
|
||||
checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
@@ -793,12 +793,6 @@ dependencies = [
|
||||
"yaml-rust2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.2"
|
||||
@@ -850,6 +844,12 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cpubits"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -940,12 +940,16 @@ checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.5.5"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core 0.6.4",
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
"getrandom 0.4.3",
|
||||
"hybrid-array",
|
||||
"num-traits",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -967,7 +971,9 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"hybrid-array",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1007,6 +1013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1075,24 +1082,14 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.7.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
||||
dependencies = [
|
||||
"const-oid 0.9.6",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"const-oid",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -1109,7 +1106,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
@@ -1121,7 +1117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.1",
|
||||
"const-oid 0.10.2",
|
||||
"const-oid",
|
||||
"crypto-common 0.2.2",
|
||||
"ctutils",
|
||||
]
|
||||
@@ -1160,16 +1156,17 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.16.9"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"digest 0.10.7",
|
||||
"der",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
"signature 2.2.0",
|
||||
"spki 0.7.3",
|
||||
"signature 3.0.0",
|
||||
"spki",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1178,7 +1175,7 @@ version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a"
|
||||
dependencies = [
|
||||
"pkcs8 0.11.0",
|
||||
"pkcs8",
|
||||
"signature 3.0.0",
|
||||
]
|
||||
|
||||
@@ -1208,19 +1205,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.13.8"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
|
||||
checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
"digest 0.10.7",
|
||||
"crypto-common 0.2.2",
|
||||
"digest 0.11.3",
|
||||
"ff",
|
||||
"generic-array",
|
||||
"group",
|
||||
"hybrid-array",
|
||||
"pem-rfc7468",
|
||||
"pkcs8 0.10.2",
|
||||
"rand_core 0.6.4",
|
||||
"pkcs8",
|
||||
"rand_core 0.10.1",
|
||||
"sec1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -1289,11 +1287,11 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
|
||||
checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -1677,12 +1675,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.13.0"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
|
||||
checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"rand_core 0.6.4",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -1779,16 +1777,7 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"hmac",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1879,7 +1868,9 @@ version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c"
|
||||
dependencies = [
|
||||
"subtle",
|
||||
"typenum",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2359,14 +2350,15 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
version = "0.13.2"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b"
|
||||
checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a"
|
||||
dependencies = [
|
||||
"ecdsa",
|
||||
"elliptic-curve",
|
||||
"primefield",
|
||||
"primeorder",
|
||||
"sha2 0.10.9",
|
||||
"sha2 0.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2427,9 +2419,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "0.7.0"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
|
||||
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
]
|
||||
@@ -2512,24 +2504,14 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
"der",
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2574,12 +2556,30 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primeorder"
|
||||
version = "0.13.6"
|
||||
name = "primefield"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6"
|
||||
checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4"
|
||||
dependencies = [
|
||||
"crypto-bigint",
|
||||
"crypto-common 0.2.2",
|
||||
"ff",
|
||||
"rand_core 0.10.1",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primeorder"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06"
|
||||
dependencies = [
|
||||
"elliptic-curve",
|
||||
"once_cell",
|
||||
"primefield",
|
||||
"serdect",
|
||||
"wnaf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2833,9 +2833,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.4.1"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a"
|
||||
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -2948,12 +2948,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.4.0"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2"
|
||||
checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b"
|
||||
dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"subtle",
|
||||
"crypto-bigint",
|
||||
"hmac",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3095,14 +3095,14 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "sec1"
|
||||
version = "0.7.3"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"der 0.7.10",
|
||||
"generic-array",
|
||||
"pkcs8 0.10.2",
|
||||
"ctutils",
|
||||
"der",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
@@ -3213,6 +3213,16 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serdect"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.7"
|
||||
@@ -3294,7 +3304,6 @@ version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
@@ -3303,6 +3312,10 @@ name = "signature"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5"
|
||||
dependencies = [
|
||||
"digest 0.11.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
@@ -3368,16 +3381,6 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.7.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.8.0"
|
||||
@@ -3385,7 +3388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.1",
|
||||
"der",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3523,7 +3526,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hkdf",
|
||||
"hmac 0.13.0",
|
||||
"hmac",
|
||||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
@@ -4493,6 +4496,17 @@ version = "0.57.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
||||
|
||||
[[package]]
|
||||
name = "wnaf"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"group",
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.3"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -13,13 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
Expired,
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -75,9 +208,9 @@ pub struct AccountRecord {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -170,13 +303,15 @@ pub struct AccountPage {
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
|
||||
@@ -443,14 +443,17 @@ pub struct RequestMetricPoint {
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted` 仍单列,以便状态详情展示和诊断。
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
|
||||
@@ -11,7 +11,10 @@ use gateway_core::{
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -153,8 +156,6 @@ pub struct PreparedCredentialCreate {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -573,6 +574,7 @@ pub struct ProviderQuotaWindow {
|
||||
pub window_seconds: Option<u64>,
|
||||
pub used_percent: Option<f64>,
|
||||
pub reset_at: Option<DateTime<Utc>>,
|
||||
pub limit_reached: bool,
|
||||
pub local_usage: Option<AccountUsage>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
@@ -591,6 +593,11 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
@@ -687,6 +694,8 @@ impl fmt::Debug for ProviderExport {
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,7 +18,8 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -216,8 +217,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -260,18 +264,37 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
provider
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
// 该账号降级为空额度投影,其余账号与页面状态照常返回。
|
||||
let provider = match self.providers.require(&account.provider_kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Ok(empty_quota());
|
||||
}
|
||||
};
|
||||
match provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: rolling_usage.get(&account.id).cloned(),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))
|
||||
{
|
||||
Ok(quota) => Ok(quota),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
Ok(empty_quota())
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -282,11 +305,16 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -407,7 +435,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, Utc::now());
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -441,7 +469,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, Utc::now())
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
@@ -486,31 +514,48 @@ impl AccountsService for DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
AccountStatus::Active
|
||||
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
|
||||
@@ -127,6 +127,8 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(18_000), Some(30.0)),
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
@@ -146,6 +148,7 @@ fn quota_window(
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}
|
||||
|
||||
@@ -373,7 +373,7 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
accounts: Mutex<Vec<AccountRecord>>,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
@@ -388,7 +388,7 @@ impl FakeAccountStore {
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
@@ -408,6 +408,10 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
@@ -439,28 +443,41 @@ impl FakeAccountStore {
|
||||
#[async_trait]
|
||||
impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: vec![self.account.clone()],
|
||||
total: 1,
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
summary: AccountSummary {
|
||||
total: 1,
|
||||
active: 1,
|
||||
total: accounts.len() as u64,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.account.clone());
|
||||
Ok((account_id == account.id.as_str()).then_some(account))
|
||||
.filter(|account| account.id == account_id)
|
||||
.or_else(|| {
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -488,11 +505,12 @@ impl AccountStore for FakeAccountStore {
|
||||
_: CredentialListQuery,
|
||||
) -> AdminStoreResult<CredentialPage> {
|
||||
self.record("store.list_credentials");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(CredentialPage {
|
||||
config_revision: revision(1),
|
||||
items: (provider_kind == &self.account.provider_kind)
|
||||
.then(|| self.account.clone())
|
||||
items: accounts
|
||||
.into_iter()
|
||||
.filter(|account| &account.provider_kind == provider_kind)
|
||||
.collect(),
|
||||
next_cursor: None,
|
||||
})
|
||||
@@ -504,14 +522,19 @@ impl AccountStore for FakeAccountStore {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> AdminStoreResult<Option<CredentialDetails>> {
|
||||
self.record("store.credential_details");
|
||||
Ok(
|
||||
(provider_kind == &self.account.provider_kind
|
||||
&& account_id.as_str() == self.account.id)
|
||||
.then(|| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential: self.account.clone(),
|
||||
}),
|
||||
)
|
||||
let account = self
|
||||
.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| {
|
||||
&account.provider_kind == provider_kind && account.id == account_id.as_str()
|
||||
})
|
||||
.cloned();
|
||||
Ok(account.map(|credential| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn load_credentials_for_export(
|
||||
@@ -520,21 +543,26 @@ impl AccountStore for FakeAccountStore {
|
||||
account_ids: &[ProviderAccountId],
|
||||
) -> AdminStoreResult<Vec<ProviderExportCredentialInput>> {
|
||||
self.record("store.load_credentials_for_export");
|
||||
if provider_kind != &self.account.provider_kind
|
||||
|| account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() != self.account.id)
|
||||
{
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
if accounts.iter().any(|account| {
|
||||
&account.provider_kind != provider_kind
|
||||
|| !account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() == account.id)
|
||||
}) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::NotFound,
|
||||
"test account",
|
||||
"credential not found",
|
||||
));
|
||||
}
|
||||
Ok(vec![ProviderExportCredentialInput {
|
||||
account: self.account.clone(),
|
||||
provider_material: document(),
|
||||
}])
|
||||
Ok(accounts
|
||||
.into_iter()
|
||||
.map(|account| ProviderExportCredentialInput {
|
||||
account,
|
||||
provider_material: document(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn commit_credential_import(
|
||||
@@ -818,7 +846,7 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
@@ -841,7 +869,7 @@ async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
..
|
||||
})
|
||||
));
|
||||
@@ -880,7 +908,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let expected = store.account.clone();
|
||||
let expected = store.accounts.lock().expect("accounts")[0].clone();
|
||||
accounts_service(provider.clone(), store)
|
||||
.await
|
||||
.accounts()
|
||||
@@ -922,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -958,22 +986,87 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() {
|
||||
let events = events();
|
||||
let openai = FakeProviderAdmin::new("openai", events.clone());
|
||||
openai.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_bad_xai".to_owned();
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(openai)
|
||||
.provider(failing)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("one failing quota must not fail the directory page");
|
||||
|
||||
assert_eq!(page.items.len(), 2);
|
||||
let by_id = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| (item.account.id.as_str(), item))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
let healthy = by_id.get("acct_test").expect("healthy account item");
|
||||
assert_eq!(healthy.quota.windows.len(), 1);
|
||||
assert_eq!(healthy.quota.windows[0].key, "primary");
|
||||
let degraded = by_id.get("acct_bad_xai").expect("degraded account item");
|
||||
assert!(degraded.quota.windows.is_empty());
|
||||
assert!(degraded.quota.observed_at.is_none());
|
||||
assert!(degraded.quota.provider_data.is_none());
|
||||
assert_eq!(page.summary.total, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
@@ -997,6 +1090,125 @@ async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_exp
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1012,9 +1224,12 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1122,6 +1337,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1166,9 +1383,8 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -1204,8 +1420,6 @@ fn prepared_create_with_id(
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
}
|
||||
}
|
||||
@@ -1290,9 +1504,7 @@ impl AccountProbe for FreeModelQuotaProbe {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24));
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
|
||||
@@ -91,7 +91,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -127,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -172,9 +162,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -194,7 +186,10 @@ pub struct AccountView {
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
pub token_refreshing: bool,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
@@ -214,6 +209,9 @@ pub struct AccountView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -227,6 +225,7 @@ pub struct AccountQuotaWindowView {
|
||||
pub label_display: String,
|
||||
pub used_percent: Option<f64>,
|
||||
pub used_percent_display: String,
|
||||
pub limit_reached: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_usage: Option<serde_json::Value>,
|
||||
pub reset_at_display: String,
|
||||
@@ -526,7 +525,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -544,7 +543,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -1332,9 +1331,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1365,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = status.as_str().to_owned();
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -1387,7 +1389,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
token_refreshing: false,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
access_token_expires_at: expires_at,
|
||||
@@ -1417,9 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1435,6 +1441,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
local_usage,
|
||||
provider_data: _,
|
||||
} = window;
|
||||
@@ -1446,6 +1453,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
used_percent,
|
||||
used_percent_display: used_percent
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
|
||||
limit_reached,
|
||||
local_usage: local_usage.as_ref().map(quota_local_usage),
|
||||
reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)),
|
||||
}
|
||||
@@ -1710,16 +1718,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
|
||||
@@ -784,7 +784,9 @@ pub(crate) fn dashboard_view(
|
||||
pool_summary: DashboardPoolSummaryView {
|
||||
total: provider_accounts.total,
|
||||
active: provider_accounts.active,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
expired: provider_accounts.expired,
|
||||
invalid: provider_accounts.invalid,
|
||||
quota_exhausted: provider_accounts.quota_exhausted,
|
||||
refreshing: provider_accounts.refreshing,
|
||||
disabled: provider_accounts.disabled,
|
||||
|
||||
@@ -348,7 +348,9 @@ pub struct DashboardAccountRequestBucketView {
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -331,7 +344,7 @@ mod actions {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
@@ -365,7 +378,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
|
||||
@@ -224,7 +224,9 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
|
||||
@@ -188,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData {
|
||||
}
|
||||
|
||||
/// 数据库中固定的账号运行状态。
|
||||
///
|
||||
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
|
||||
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
@@ -206,7 +208,6 @@ impl AccountAvailability {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Ready => "ready",
|
||||
Self::Cooldown => "cooldown",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Expired => "expired",
|
||||
Self::Banned => "banned",
|
||||
@@ -220,7 +221,6 @@ impl AccountAvailability {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"ready" => Some(Self::Ready),
|
||||
"cooldown" => Some(Self::Cooldown),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"expired" => Some(Self::Expired),
|
||||
"banned" => Some(Self::Banned),
|
||||
@@ -244,7 +244,6 @@ pub struct ProviderAccount {
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
@@ -274,7 +273,6 @@ impl ProviderAccount {
|
||||
revision,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Unknown,
|
||||
cooldown_until: None,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
@@ -299,11 +297,9 @@ impl ProviderAccount {
|
||||
mut self,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.availability = availability;
|
||||
self.cooldown_until = cooldown_until;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -373,11 +369,6 @@ impl ProviderAccount {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cooldown_until(&self) -> Option<SystemTime> {
|
||||
self.cooldown_until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn access_token_expires_at(&self) -> Option<SystemTime> {
|
||||
self.access_token_expires_at
|
||||
@@ -398,7 +389,6 @@ impl ProviderAccount {
|
||||
pub fn is_schedulable(&self, now: SystemTime) -> bool {
|
||||
let available = match self.availability {
|
||||
AccountAvailability::Ready => true,
|
||||
AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
@@ -588,6 +578,8 @@ pub struct QuotaObservation {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: Option<OpaqueProviderData>,
|
||||
pub observed_at: Option<SystemTime>,
|
||||
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
|
||||
pub limit_reached: Option<bool>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for QuotaObservation {
|
||||
@@ -614,9 +606,9 @@ pub struct AccountStateChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub reason: Option<String>,
|
||||
pub cooldown_until: Option<SystemTime>,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口。
|
||||
@@ -756,6 +748,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub quota_limit_reached: bool,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -903,6 +896,7 @@ impl AccountRuntimeSignals {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
self.quota_limit_reached = quota.limit_reached;
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -924,14 +918,20 @@ impl AccountRuntimeSignals {
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl AccountQuotaSignals {
|
||||
#[must_use]
|
||||
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
|
||||
pub const fn new(
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -944,6 +944,12 @@ impl AccountQuotaSignals {
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
|
||||
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实与可重建运行信号的请求级组合。
|
||||
|
||||
@@ -635,6 +635,13 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
/// 生命周期与 credential revision 无关;不要求调用方持有 revision。
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -24,7 +24,7 @@ fn account(id: &str) -> ProviderAccount {
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
|
||||
@@ -35,6 +35,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -110,31 +111,27 @@ fn availability_should_round_trip_all_database_values() {
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_exhausted")
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_disabled")
|
||||
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -158,33 +155,11 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account =
|
||||
account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready, None);
|
||||
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
|
||||
|
||||
assert!(!account.is_schedulable(SystemTime::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_become_schedulable_at_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account =
|
||||
account("acct_cooldown").with_runtime_state(true, AccountAvailability::Cooldown, Some(now));
|
||||
|
||||
assert!(account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_remain_isolated_before_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account = account("acct_cooldown").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::Cooldown,
|
||||
Some(now + Duration::from_secs(1)),
|
||||
);
|
||||
|
||||
assert!(!account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_prefer_lower_inflight_count() {
|
||||
let candidates = vec![
|
||||
@@ -311,10 +286,15 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(
|
||||
Some(reset_at),
|
||||
Some(75),
|
||||
false,
|
||||
)));
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -629,3 +609,48 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
// availability 终态(Banned/Expired/Invalid)始终排除,
|
||||
// 不受 quota 信号或 skip_exhausted 放宽影响。
|
||||
let mut banned = candidate("acct_banned", 0, Some(100));
|
||||
banned.account = banned
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Banned);
|
||||
let mut expired = candidate("acct_expired", 0, Some(100));
|
||||
expired.account = expired
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Expired);
|
||||
let mut invalid = candidate("acct_invalid", 0, Some(100));
|
||||
invalid.account = invalid
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Invalid);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![banned, expired, invalid, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
|
||||
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
|
||||
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
|
||||
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![exhausted, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountStore;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 已完成连接、迁移与 cooldown hydration 的 Store 能力集合。
|
||||
/// 已完成连接、迁移与 hydration 的 Store 能力集合。
|
||||
pub struct StoreBundle {
|
||||
admin_ports: AdminStorePorts,
|
||||
core_ports: CoreStorePorts,
|
||||
@@ -61,12 +64,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let account_store = Arc::new(redis::CooldownCachingProviderAccountStore::new(
|
||||
provider_accounts,
|
||||
cooldowns.clone(),
|
||||
));
|
||||
let hydrated = account_store.hydrate(std::time::SystemTime::now()).await;
|
||||
tracing::info!(hydrated, "账号 cooldown Redis 热缓存重建完成");
|
||||
let account_store: Arc<dyn ProviderAccountStore> = provider_accounts;
|
||||
|
||||
let credential_leases =
|
||||
redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?;
|
||||
@@ -88,7 +86,10 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
)?);
|
||||
|
||||
let admin_ports = AdminStorePorts::new(
|
||||
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
|
||||
Arc::new(postgres::PgAdminAccountStore::new(
|
||||
pool.clone(),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminAuthStoreAdapter {
|
||||
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
|
||||
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
|
||||
@@ -101,6 +102,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
Arc::new(postgres::PgAdminObservabilityStore::new(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminSettingsStoreAdapter {
|
||||
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 八张终态业务表的 PostgreSQL adapters。
|
||||
//! 九张终态业务表的 PostgreSQL adapters。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::{PgPool, postgres::PgPoolOptions};
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -7,12 +12,73 @@ use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepositor
|
||||
#[derive(Clone)]
|
||||
pub struct PgObservabilityRepository {
|
||||
pool: PgPool,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgObservabilityRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,用 Redis 429 冷却对每个非终态账号重新归类——
|
||||
/// 冷却中 → `rate_limited`(限流中);否则按 `quota_limit_reached` 归
|
||||
/// `quota_exhausted`(配额耗尽)或 `active`。冷却不可用时不修正(限流中计入配额耗尽)。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
let mut active = 0_u64;
|
||||
let mut rate_limited = 0_u64;
|
||||
let mut quota_exhausted = 0_u64;
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > SystemTime::now())
|
||||
.is_some();
|
||||
if cooling {
|
||||
rate_limited += 1;
|
||||
} else if quota_reached {
|
||||
quota_exhausted += 1;
|
||||
} else {
|
||||
active += 1;
|
||||
}
|
||||
}
|
||||
// 终态计数(expired/invalid/disabled/banned)保持 SQL 原值。
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,9 +94,13 @@ pub struct PgAdminObservabilityStore {
|
||||
|
||||
impl PgAdminObservabilityStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository: PgObservabilityRepository::new(pool),
|
||||
repository: PgObservabilityRepository::new(pool, cooldowns),
|
||||
runtime_signals,
|
||||
}
|
||||
}
|
||||
@@ -62,11 +132,13 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, provider_accounts) = futures::try_join!(
|
||||
let (requests, attempts) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
provider_account_metrics(&self.pool, range.end),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
|
||||
@@ -263,7 +263,9 @@ pub(crate) const fn admin_account_pool_metrics(
|
||||
enabled: metrics.enabled,
|
||||
unavailable: metrics.unavailable,
|
||||
active: metrics.active,
|
||||
rate_limited: metrics.rate_limited,
|
||||
expired: metrics.expired,
|
||||
invalid: metrics.invalid,
|
||||
quota_exhausted: metrics.quota_exhausted,
|
||||
refreshing: None,
|
||||
disabled: metrics.disabled,
|
||||
|
||||
@@ -366,7 +366,9 @@ pub struct ProviderAccountMetrics {
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
|
||||
@@ -348,29 +348,28 @@ pub(crate) async fn provider_account_metrics(
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"with normalized as (
|
||||
select enabled,
|
||||
select enabled, availability, quota_limit_reached,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability = 'invalid' then 'invalid'
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when quota_limit_reached then 'rate_limited'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where status <> 'active')::bigint as unavailable,
|
||||
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'invalid')::bigint as invalid,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
@@ -385,7 +384,9 @@ pub(crate) async fn provider_account_metrics(
|
||||
enabled: unsigned(&row, "enabled")?,
|
||||
unavailable: unsigned(&row, "unavailable")?,
|
||||
active: unsigned(&row, "active")?,
|
||||
rate_limited: unsigned(&row, "rate_limited")?,
|
||||
expired: unsigned(&row, "expired")?,
|
||||
invalid: unsigned(&row, "invalid")?,
|
||||
quota_exhausted: unsigned(&row, "quota_exhausted")?,
|
||||
disabled: unsigned(&row, "disabled")?,
|
||||
banned: unsigned(&row, "banned")?,
|
||||
@@ -402,13 +403,7 @@ pub(crate) async fn active_provider_account_ids(
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
@@ -430,6 +425,33 @@ pub(crate) async fn active_provider_account_ids(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询非终态候选账号:id + credential revision + 是否 quota 触顶。
|
||||
/// 仅含 ready/unknown 等可被 Redis 冷却影响的账号,终态(禁用/封禁/过期/无效)由
|
||||
/// [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability in ('ready', 'unknown')
|
||||
order by id",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
//! `gateway-admin` 账号端口的 PostgreSQL adapter。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。
|
||||
@@ -11,19 +17,49 @@ pub struct PgAdminAccountStore {
|
||||
accounts: PgProviderAccountRepository,
|
||||
observability: PgObservabilityRepository,
|
||||
control_plane: PgControlPlaneRepository,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgAdminAccountStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
pub fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self {
|
||||
pool: pool.clone(),
|
||||
accounts: PgProviderAccountRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone(), cooldowns.clone()),
|
||||
control_plane: PgControlPlaneRepository::new(pool),
|
||||
cooldowns,
|
||||
}
|
||||
}
|
||||
|
||||
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
|
||||
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
|
||||
async fn load_rate_limited_until(
|
||||
&self,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let mut map = BTreeMap::new();
|
||||
for account in accounts {
|
||||
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
|
||||
continue;
|
||||
};
|
||||
if cooldown.credential_revision().get() != account.credential_revision.get() {
|
||||
continue;
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until > SystemTime::now() {
|
||||
map.insert(account.id.clone(), until);
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
async fn usage_observations(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
@@ -242,11 +278,17 @@ impl AccountStore for PgAdminAccountStore {
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let now = Utc::now();
|
||||
let summary = admin_account_summary(&accounts, now);
|
||||
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
|
||||
let rate_limited_until = self
|
||||
.load_rate_limited_until(&accounts)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
|
||||
let mut items = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let status = admin_account_status(&account, now);
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
let status = admin_account_status(&account, now, until);
|
||||
account_matches_admin_query(&account, status, &query).then_some(
|
||||
AdminAccountListItem {
|
||||
account,
|
||||
|
||||
@@ -32,8 +32,6 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
next_refresh_at: account.account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.account.enabled(),
|
||||
availability: account.account.availability(),
|
||||
availability_reason: None,
|
||||
cooldown_until: account.account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -165,7 +163,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.map(|account| account.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let rows = sqlx::query(
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at \
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at, \
|
||||
quota_limit_reached \
|
||||
from provider_accounts \
|
||||
where id = any($1) and provider_quota_json is not null",
|
||||
)
|
||||
@@ -193,12 +192,16 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
let observed_at = row
|
||||
.try_get::<DateTime<Utc>, _>("quota_observed_at")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let limit_reached = row
|
||||
.try_get::<bool, _>("quota_limit_reached")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
Ok(QuotaObservation {
|
||||
account_id: CoreProviderAccountId::new(account_id)
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
expected_revision: revision,
|
||||
quota: Some(quota),
|
||||
observed_at: Some(observed_at.into()),
|
||||
limit_reached: Some(limit_reached),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -225,6 +228,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
Revision::new(observation.expected_revision.get()).map_err(core_store_error)?,
|
||||
quota,
|
||||
observation.observed_at.map(DateTime::<Utc>::from),
|
||||
observation.limit_reached,
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
@@ -242,9 +246,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
expected_revision: Revision::new(change.expected_revision.get())
|
||||
.map_err(core_store_error)?,
|
||||
availability: change.availability,
|
||||
availability_reason: change.reason,
|
||||
cooldown_until: change.cooldown_until.map(DateTime::<Utc>::from),
|
||||
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
|
||||
message: change.message,
|
||||
})
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
use gateway_admin::model::accounts::AccountStatusSignals;
|
||||
|
||||
pub(crate) struct AdminAccountListItem {
|
||||
pub(crate) account: ProviderAccountSummary,
|
||||
pub(crate) status: AdminAccountStatus,
|
||||
@@ -58,50 +60,48 @@ pub(crate) fn account_matches_admin_query(
|
||||
pub(crate) fn admin_account_status(
|
||||
account: &ProviderAccountSummary,
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AdminAccountStatus {
|
||||
if !account.enabled {
|
||||
AdminAccountStatus::Disabled
|
||||
} else if account.availability == AccountAvailability::Banned {
|
||||
AdminAccountStatus::Banned
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == AccountAvailability::Expired
|
||||
|| account.availability == AccountAvailability::Invalid
|
||||
|| account
|
||||
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AdminAccountStatus::Expired
|
||||
} else {
|
||||
AdminAccountStatus::Active
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: account.quota_limit_reached,
|
||||
rate_limited_until,
|
||||
}
|
||||
.derive()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_summary(
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: &BTreeMap<String, SystemTime>,
|
||||
) -> AccountSummary {
|
||||
let mut summary = AccountSummary {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
active: 0,
|
||||
normal: 0,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
};
|
||||
for account in accounts {
|
||||
match admin_account_status(account, now) {
|
||||
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
match admin_account_status(account, now, until) {
|
||||
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
|
||||
AdminAccountStatus::QuotaExhausted => {
|
||||
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Expired
|
||||
| AdminAccountStatus::Disabled
|
||||
| AdminAccountStatus::Banned => {
|
||||
summary.unavailable = summary.unavailable.saturating_add(1);
|
||||
AdminAccountStatus::RateLimited => {
|
||||
summary.rate_limited = summary.rate_limited.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Disabled => {
|
||||
summary.disabled = summary.disabled.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Error => summary.error = summary.error.saturating_add(1),
|
||||
}
|
||||
}
|
||||
summary
|
||||
@@ -118,9 +118,7 @@ pub(crate) fn sort_admin_account_items(
|
||||
items.sort_by(|left, right| {
|
||||
let ordering = match sort.field {
|
||||
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
|
||||
AdminAccountSortField::Status => {
|
||||
admin_account_status_name(left.status).cmp(admin_account_status_name(right.status))
|
||||
}
|
||||
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
|
||||
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
|
||||
AdminAccountSortField::Usage => left
|
||||
.usage
|
||||
@@ -150,16 +148,6 @@ pub(crate) fn sort_admin_account_items(
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
|
||||
match status {
|
||||
AdminAccountStatus::Active => "active",
|
||||
AdminAccountStatus::Expired => "expired",
|
||||
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
AdminAccountStatus::Disabled => "disabled",
|
||||
AdminAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_record(
|
||||
summary: ProviderAccountSummary,
|
||||
) -> AdminStoreResult<AccountRecord> {
|
||||
@@ -184,9 +172,8 @@ pub(crate) fn admin_account_record(
|
||||
next_refresh_at: summary.next_refresh_at,
|
||||
enabled: summary.enabled,
|
||||
availability: summary.availability,
|
||||
availability_reason: summary.availability_reason,
|
||||
cooldown_until: summary.cooldown_until,
|
||||
availability_observed_at: summary.availability_observed_at,
|
||||
last_error_message: summary.last_error_message,
|
||||
quota_observed_at: summary.quota_observed_at,
|
||||
created_at: summary.created_at,
|
||||
updated_at: summary.updated_at,
|
||||
@@ -211,8 +198,6 @@ pub(crate) fn prepared_account(
|
||||
next_refresh_at: credential.next_refresh_at,
|
||||
enabled: credential.enabled,
|
||||
availability: credential.availability,
|
||||
availability_reason: credential.availability_reason,
|
||||
cooldown_until: credential.cooldown_until,
|
||||
availability_observed_at: credential.availability_observed_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
str::FromStr,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -27,6 +27,7 @@ pub trait ProviderAccountRepository: Send + Sync {
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
) -> StoreResult<bool>;
|
||||
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool>;
|
||||
}
|
||||
@@ -93,8 +94,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
"select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, availability_observed_at,
|
||||
quota_observed_at, created_at, updated_at
|
||||
availability_observed_at,
|
||||
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
|
||||
from provider_accounts
|
||||
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
|
||||
order by provider_kind, name, id",
|
||||
@@ -114,11 +115,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, availability_reason, cooldown_until, provider_quota_json,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, null, $15, null, $16, null, now(), greatest(now(), $16)
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
)",
|
||||
)
|
||||
.bind(account.id)
|
||||
@@ -135,7 +136,6 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(account.availability.as_str())
|
||||
.bind(account.cooldown_until)
|
||||
.bind(account.availability_observed_at)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
@@ -210,22 +210,24 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = case when enabled then $3 else availability end,
|
||||
availability_reason = case when enabled then $4 else availability_reason end,
|
||||
cooldown_until = case when enabled then $5 else cooldown_until end,
|
||||
availability_observed_at = case
|
||||
when enabled then $6
|
||||
when enabled then $4
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $6) else updated_at end
|
||||
last_error_message = case
|
||||
when enabled then
|
||||
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
|
||||
else last_error_message
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $4) else updated_at end
|
||||
where id = $1 and credential_revision = $2
|
||||
and (availability_observed_at is null or availability_observed_at <= $6)",
|
||||
and (availability_observed_at is null or availability_observed_at <= $4)",
|
||||
)
|
||||
.bind(update.account_id)
|
||||
.bind(to_i64(update.expected_revision.get())?)
|
||||
.bind(update.availability.as_str())
|
||||
.bind(update.availability_reason)
|
||||
.bind(update.cooldown_until)
|
||||
.bind(update.availability_observed_at)
|
||||
.bind(update.message.as_deref())
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("apply provider account state"))?;
|
||||
@@ -251,6 +253,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "account_id", account_id)?;
|
||||
if quota.is_some() != observed_at.is_some() {
|
||||
@@ -262,6 +265,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set provider_quota_json = $3, quota_observed_at = $4,
|
||||
quota_limit_reached = coalesce($5, quota_limit_reached),
|
||||
updated_at = greatest(now(), coalesce($4, now()))
|
||||
where id = $1 and credential_revision = $2
|
||||
and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)",
|
||||
@@ -270,6 +274,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(to_i64(expected_revision.get())?)
|
||||
.bind(quota.map(|value| value.as_value()))
|
||||
.bind(observed_at)
|
||||
.bind(limit_reached)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
|
||||
@@ -466,11 +471,11 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, availability_reason, cooldown_until, provider_quota_json,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, $15, $16, null, $17, null, now(), greatest(now(), $17)
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
)
|
||||
on conflict (
|
||||
provider_kind,
|
||||
@@ -488,8 +493,6 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
next_refresh_at = excluded.next_refresh_at,
|
||||
enabled = excluded.enabled,
|
||||
availability = excluded.availability,
|
||||
availability_reason = excluded.availability_reason,
|
||||
cooldown_until = excluded.cooldown_until,
|
||||
provider_quota_json = null,
|
||||
availability_observed_at = excluded.availability_observed_at,
|
||||
quota_observed_at = null,
|
||||
@@ -510,8 +513,6 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(account.availability.as_str())
|
||||
.bind(&account.availability_reason)
|
||||
.bind(account.cooldown_until)
|
||||
.bind(account.availability_observed_at)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
@@ -565,16 +566,6 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
when availability <> 'quota_exhausted' then 'ready'
|
||||
else availability
|
||||
end,
|
||||
availability_reason = case
|
||||
when not enabled then availability_reason
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else availability_reason
|
||||
end,
|
||||
cooldown_until = case
|
||||
when not enabled then cooldown_until
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else cooldown_until
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when not enabled then availability_observed_at
|
||||
when availability <> 'quota_exhausted' then now()
|
||||
|
||||
@@ -70,26 +70,6 @@ pub(crate) fn parse_availability(value: &str) -> StoreResult<AccountAvailability
|
||||
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_availability_cooldown(
|
||||
availability: AccountAvailability,
|
||||
has_cooldown_until: bool,
|
||||
) -> StoreResult<()> {
|
||||
if availability == AccountAvailability::Cooldown && !has_cooldown_until {
|
||||
return Err(invalid("cooldown availability requires cooldown_until"));
|
||||
}
|
||||
if has_cooldown_until
|
||||
&& !matches!(
|
||||
availability,
|
||||
AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted
|
||||
)
|
||||
{
|
||||
return Err(invalid(
|
||||
"cooldown_until is only allowed for cooldown or quota_exhausted availability",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountSummary {
|
||||
pub id: String,
|
||||
@@ -106,10 +86,12 @@ pub struct ProviderAccountSummary {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -151,8 +133,6 @@ pub struct NewProviderAccount {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -173,7 +153,6 @@ impl fmt::Debug for NewProviderAccount {
|
||||
.field("name", &self.name)
|
||||
.field("enabled", &self.enabled)
|
||||
.field("availability", &self.availability)
|
||||
.field("cooldown_until", &self.cooldown_until)
|
||||
.field("provider_credentials_json", &"[REDACTED]")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
@@ -189,7 +168,6 @@ impl NewProviderAccount {
|
||||
if !self.has_refresh_token && self.next_refresh_at.is_some() {
|
||||
return Err(invalid("next_refresh_at requires a refresh token"));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
validate_object_size(
|
||||
"provider_credentials_json",
|
||||
&self.provider_credentials_json,
|
||||
@@ -321,11 +299,11 @@ impl fmt::Debug for ProviderCredentialUpdate {
|
||||
pub struct ProviderAccountObservation {
|
||||
pub account_id: String,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub provider_quota_json: Option<JsonObject>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -333,15 +311,14 @@ pub struct ProviderAccountStateUpdate {
|
||||
pub account_id: String,
|
||||
pub expected_revision: Revision,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl ProviderAccountStateUpdate {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -349,7 +326,6 @@ impl ProviderAccountStateUpdate {
|
||||
impl ProviderAccountObservation {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and quota_observed_at must agree"));
|
||||
}
|
||||
@@ -363,14 +339,14 @@ impl ProviderAccountObservation {
|
||||
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, provider_quota_json,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts where id = $1";
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_reason, cooldown_until, provider_quota_json,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts
|
||||
where id = any($1::text[]) and provider_kind = $2
|
||||
@@ -420,11 +396,7 @@ pub(crate) fn core_account_from_summary(
|
||||
summary.upstream_account_id,
|
||||
summary.plan_type,
|
||||
)
|
||||
.with_runtime_state(
|
||||
summary.enabled,
|
||||
summary.availability,
|
||||
summary.cooldown_until.map(Into::into),
|
||||
)
|
||||
.with_runtime_state(summary.enabled, summary.availability)
|
||||
.with_refresh_schedule(
|
||||
summary.has_refresh_token,
|
||||
summary.next_refresh_at.map(Into::into),
|
||||
@@ -474,10 +446,10 @@ pub(crate) fn account_summary_from_row(
|
||||
next_refresh_at: get(&row, "next_refresh_at")?,
|
||||
enabled: get(&row, "enabled")?,
|
||||
availability: parse_availability(&availability)?,
|
||||
availability_reason: get(&row, "availability_reason")?,
|
||||
cooldown_until: get(&row, "cooldown_until")?,
|
||||
availability_observed_at: get(&row, "availability_observed_at")?,
|
||||
quota_observed_at: get(&row, "quota_observed_at")?,
|
||||
quota_limit_reached: get(&row, "quota_limit_reached")?,
|
||||
last_error_message: get(&row, "last_error_message")?,
|
||||
created_at: get(&row, "created_at")?,
|
||||
updated_at: get(&row, "updated_at")?,
|
||||
})
|
||||
|
||||
@@ -1,22 +1,13 @@
|
||||
//! 请求调度用的可丢失账号 cooldown Redis 存储。
|
||||
|
||||
use std::{sync::Arc, time::SystemTime};
|
||||
//! 请求调度用的可丢失 Provider 级 cooldown Redis 存储。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
},
|
||||
error::StoreError as CoreStoreError,
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError, ProviderStoreErrorKind,
|
||||
},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
|
||||
@@ -85,6 +76,8 @@ pub trait CredentialCooldownRepository: Send + Sync {
|
||||
provider_account_id: &str,
|
||||
through_revision: Revision,
|
||||
) -> StoreResult<bool>;
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
async fn delete_account_cooldowns(&self, provider_account_id: &str) -> StoreResult<bool>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -227,6 +220,50 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
self.invalidate_at_key(self.key(provider_account_id)?, through_revision)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_account_cooldowns(&self, provider_account_id: &str) -> StoreResult<bool> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
provider_account_id,
|
||||
)?;
|
||||
// 账号删除:清除该账号的 account key 与全部 model-scoped key。
|
||||
// 用 SCAN 精确匹配命名空间内该账号前缀,避免 KEYS 阻塞。
|
||||
let mut connection = self.connection.clone();
|
||||
let account_key = self.key(provider_account_id)?;
|
||||
let mut keys = vec![account_key.clone()];
|
||||
let pattern = format!(
|
||||
"{}:account:{}:cooldown:*",
|
||||
self.namespace,
|
||||
resource_fingerprint("credential cooldown", provider_account_id)?
|
||||
);
|
||||
let mut cursor = 0_i64;
|
||||
loop {
|
||||
let (next, found): (i64, Vec<String>) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(&pattern)
|
||||
.arg("COUNT")
|
||||
.arg(100)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("scan account cooldown keys"))?;
|
||||
keys.extend(found);
|
||||
cursor = next;
|
||||
if cursor == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let removed: i64 = redis::cmd("DEL")
|
||||
.arg(keys)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("delete account cooldown keys"))?;
|
||||
Ok(removed > 0)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
@@ -350,216 +387,17 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_unavailable("clear scoped credential cooldown"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号状态写入到可丢失 cooldown cache 的桥接 adapter。
|
||||
pub struct CooldownCachingProviderAccountStore {
|
||||
authoritative: Arc<dyn ProviderAccountStore>,
|
||||
cooldowns: Arc<dyn CredentialCooldownRepository>,
|
||||
}
|
||||
|
||||
impl CooldownCachingProviderAccountStore {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
authoritative: Arc<dyn ProviderAccountStore>,
|
||||
cooldowns: Arc<dyn CredentialCooldownRepository>,
|
||||
) -> Self {
|
||||
Self {
|
||||
authoritative,
|
||||
cooldowns,
|
||||
}
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
self.delete_account_cooldowns(account_id.as_str())
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("clear all credential cooldowns"))
|
||||
})
|
||||
}
|
||||
|
||||
/// 启动时从 PostgreSQL 尽力重建仍有效的 cooldown cache。
|
||||
pub async fn hydrate(&self, now: SystemTime) -> u64 {
|
||||
let Ok(accounts) = self.authoritative.list_accounts().await else {
|
||||
return 0;
|
||||
};
|
||||
let mut hydrated = 0_u64;
|
||||
for account in accounts {
|
||||
let Some(cooldown) = projection(
|
||||
account.id(),
|
||||
account.revision(),
|
||||
account.availability(),
|
||||
account.cooldown_until(),
|
||||
now,
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if matches!(
|
||||
self.cooldowns.cache_credential_cooldown(&cooldown).await,
|
||||
Ok(true)
|
||||
) {
|
||||
hydrated = hydrated.saturating_add(1);
|
||||
}
|
||||
}
|
||||
hydrated
|
||||
}
|
||||
|
||||
async fn mirror(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) {
|
||||
let Some(cooldown) = projection(
|
||||
account_id,
|
||||
credential_revision,
|
||||
availability,
|
||||
cooldown_until,
|
||||
SystemTime::now(),
|
||||
) else {
|
||||
let Ok(revision) = Revision::new(credential_revision.get()) else {
|
||||
return;
|
||||
};
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account_id.as_str(), revision)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let _ = self.cooldowns.cache_credential_cooldown(&cooldown).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for CooldownCachingProviderAccountStore {
|
||||
async fn create_account(&self, account: NewProviderAccount) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.create_account(account).await
|
||||
}
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.get_account(account).await
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.list_accounts().await
|
||||
}
|
||||
|
||||
async fn list_for_provider(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
self.authoritative.list_for_provider(provider).await
|
||||
}
|
||||
|
||||
async fn load_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
self.authoritative
|
||||
.load_credential(account, expected_revision)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
self.authoritative.load_current_credential(account).await
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
) -> Result<CredentialCasOutcome, CoreStoreError> {
|
||||
let account_id = update.account_id().clone();
|
||||
let outcome = self
|
||||
.authoritative
|
||||
.compare_and_swap_credential(update)
|
||||
.await?;
|
||||
if let CredentialCasOutcome::Updated(credential_revision) = outcome
|
||||
&& let Ok(revision) = Revision::new(credential_revision.get())
|
||||
{
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account_id.as_str(), revision)
|
||||
.await;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn get_quotas(
|
||||
&self,
|
||||
accounts: &[ProviderAccountId],
|
||||
) -> Result<Vec<QuotaObservation>, CoreStoreError> {
|
||||
self.authoritative.get_quotas(accounts).await
|
||||
}
|
||||
|
||||
async fn compare_and_swap_quota(
|
||||
&self,
|
||||
observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
self.authoritative.compare_and_swap_quota(observation).await
|
||||
}
|
||||
|
||||
async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), CoreStoreError> {
|
||||
let account_id = change.account_id.clone();
|
||||
let credential_revision = change.expected_revision;
|
||||
let availability = change.availability;
|
||||
let cooldown_until = change.cooldown_until;
|
||||
self.authoritative.apply_state_change(change).await?;
|
||||
let Some(account) = self.authoritative.get_account(&account_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.mirror(
|
||||
&account_id,
|
||||
credential_revision,
|
||||
availability,
|
||||
cooldown_until,
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.update_account(update).await
|
||||
}
|
||||
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
enabled: bool,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.set_enabled(account, enabled).await
|
||||
}
|
||||
|
||||
async fn delete_account(&self, account: &ProviderAccountId) -> Result<(), CoreStoreError> {
|
||||
self.authoritative.delete_account(account).await?;
|
||||
if let Ok(through_revision) = Revision::new(u64::MAX) {
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.invalidate_credential_cooldown(account.as_str(), through_revision)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn projection(
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
) -> Option<CredentialCooldown> {
|
||||
let cooldown_until = cooldown_until
|
||||
.filter(|_| availability == AccountAvailability::Cooldown)
|
||||
.filter(|until| *until > now)
|
||||
.map(DateTime::<Utc>::from)?;
|
||||
Some(CredentialCooldown {
|
||||
provider_account_id: account_id.as_str().to_owned(),
|
||||
credential_revision: Revision::new(credential_revision.get()).ok()?,
|
||||
cooldown_until,
|
||||
})
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> StoreError {
|
||||
|
||||
@@ -512,6 +512,7 @@ impl RedisProviderLeaseCoordinator {
|
||||
last_started_at: signal.last_started_at.map(Into::into),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -21,6 +21,8 @@ mod runtime_settings;
|
||||
mod snapshot;
|
||||
mod snapshots;
|
||||
|
||||
static TEST_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
|
||||
|
||||
pub(super) struct TestDatabase {
|
||||
admin: PgPool,
|
||||
pub(super) pool: PgPool,
|
||||
@@ -56,28 +58,10 @@ impl TestDatabase {
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect isolated test schema");
|
||||
let mut migration = pool.begin().await.expect("begin terminal migration");
|
||||
sqlx::raw_sql(include_str!("../../../../migrations/0001_initial.sql"))
|
||||
.execute(&mut *migration)
|
||||
TEST_MIGRATOR
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("apply terminal migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0002_snapshot_provider_account_identity.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply snapshot migration");
|
||||
sqlx::raw_sql(include_str!("../../../../migrations/0003_s3_backup.sql"))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply backup migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0004_drop_provider_accounts_id_kind_uq.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply index drop migration");
|
||||
migration.commit().await.expect("commit terminal migration");
|
||||
.expect("apply test migrations");
|
||||
Some(Self {
|
||||
admin,
|
||||
pool,
|
||||
@@ -172,24 +156,19 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (10, 4));
|
||||
assert_eq!((first_table_count, migration_count), (10, 1));
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = [
|
||||
include_str!("../../../../migrations/0001_initial.sql"),
|
||||
include_str!("../../../../migrations/0002_snapshot_provider_account_identity.sql"),
|
||||
include_str!("../../../../migrations/0003_s3_backup.sql"),
|
||||
include_str!("../../../../migrations/0004_drop_provider_accounts_id_kind_uq.sql"),
|
||||
]
|
||||
.into_iter()
|
||||
.flat_map(str::lines)
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
let transaction_statements = TEST_MIGRATOR
|
||||
.iter()
|
||||
.flat_map(|migration| migration.sql.as_str().lines())
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
|
||||
assert_eq!(transaction_statements, 0);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
.expect("persist opaque response IDs");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let records = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
@@ -122,33 +122,32 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability,
|
||||
availability_reason, cooldown_until,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_available', 'openai', 'available', 'user-available', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
$1, $1, $1),
|
||||
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted',
|
||||
$1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
|
||||
null, null, $1, $1, $1),
|
||||
$1, $1, $1),
|
||||
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
|
||||
null, null, $1, $1, $1)",
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed account metric states");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
@@ -194,7 +193,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
.expect("seed calculated billing facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let facts = repository
|
||||
.usage_calculated_billing_facts(range, UsageRecordFilter::default())
|
||||
@@ -208,7 +207,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
let facts = store
|
||||
.usage_calculated_billing_facts(
|
||||
admin_observability::TimeRange::new(range.start, range.end)
|
||||
@@ -236,7 +235,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
let range =
|
||||
admin_observability::TimeRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("admin observability range");
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
|
||||
let dashboard = store
|
||||
.dashboard_summary(range)
|
||||
@@ -525,7 +524,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("seed observability facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let dashboard = repository
|
||||
.dashboard_summary(range)
|
||||
|
||||
@@ -28,9 +28,9 @@ use gateway_store::{
|
||||
postgres::{
|
||||
AdminAuditActorKind, AdminAuditEvent, DeleteProviderAccounts, ImportProviderAccounts,
|
||||
NewProviderAccount, PgAdminAccountStore, PgProviderAccountRepository,
|
||||
ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountObservation,
|
||||
ProviderAccountRepository, ProviderCredentialUpdate, RotateProviderAccount,
|
||||
SetProviderAccountEnabled, UpdateProviderAccount,
|
||||
ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountRepository,
|
||||
ProviderCredentialUpdate, RotateProviderAccount, SetProviderAccountEnabled,
|
||||
UpdateProviderAccount,
|
||||
},
|
||||
};
|
||||
use serde_json::json;
|
||||
@@ -74,6 +74,7 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -140,6 +141,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
};
|
||||
assert_eq!(
|
||||
repository
|
||||
@@ -158,11 +160,10 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: Some(newer),
|
||||
observed_at: newer,
|
||||
})
|
||||
.await
|
||||
@@ -170,11 +171,10 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
assert!(
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: older,
|
||||
})
|
||||
.await
|
||||
@@ -202,6 +202,61 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let account_id = ProviderAccountId::new("acct_state_message").expect("account ID");
|
||||
repository
|
||||
.insert_provider_account(account(account_id.as_str(), "user-state-message"))
|
||||
.await
|
||||
.expect("insert state message fixture");
|
||||
let revision = CredentialRevision::new(1).expect("revision");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: Some("upstream returned 402 payment required".to_owned()),
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist error message");
|
||||
let stored: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read error message");
|
||||
assert_eq!(
|
||||
stored.as_deref(),
|
||||
Some("upstream returned 402 payment required")
|
||||
);
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("recover account");
|
||||
let cleared: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read cleared error message");
|
||||
assert_eq!(cleared, None);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_usage() {
|
||||
let Some(database) = TestDatabase::create("provider_account_terminal_list").await else {
|
||||
@@ -233,6 +288,21 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("insert account list fixture");
|
||||
}
|
||||
repository
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({"rate_limit": {"limit_reached": true}})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed rate-limited account");
|
||||
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
@@ -291,7 +361,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("seed charlie usage");
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let usage_page = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
@@ -309,14 +379,18 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(usage_page.config_revision.get(), 1);
|
||||
assert_eq!(usage_page.total, 6);
|
||||
assert_eq!(usage_page.summary.total, 6);
|
||||
assert_eq!(usage_page.summary.active, 2);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 1);
|
||||
assert_eq!(usage_page.summary.unavailable, 3);
|
||||
assert_eq!(usage_page.summary.normal, 1);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 2);
|
||||
assert_eq!(usage_page.summary.rate_limited, 0);
|
||||
assert_eq!(usage_page.summary.disabled, 1);
|
||||
assert_eq!(usage_page.summary.error, 2);
|
||||
assert_eq!(
|
||||
usage_page.summary.total,
|
||||
usage_page.summary.active
|
||||
usage_page.summary.normal
|
||||
+ usage_page.summary.quota_exhausted
|
||||
+ usage_page.summary.unavailable
|
||||
+ usage_page.summary.rate_limited
|
||||
+ usage_page.summary.disabled
|
||||
+ usage_page.summary.error
|
||||
);
|
||||
assert_eq!(
|
||||
usage_page
|
||||
@@ -356,7 +430,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("Provider kind")),
|
||||
search: Some("ALPHA@EXAMPLE".to_owned()),
|
||||
status: Some(AccountStatus::Active),
|
||||
status: Some(AccountStatus::Normal),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
@@ -366,18 +440,20 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(filtered.items[0].id, "acct_alpha");
|
||||
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
|
||||
|
||||
let banned = store
|
||||
let error_accounts = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: Some(AccountStatus::Banned),
|
||||
status: Some(AccountStatus::Error),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("filter banned accounts");
|
||||
assert_eq!(banned.items[0].id, "acct_beta");
|
||||
.expect("filter error accounts");
|
||||
assert_eq!(error_accounts.items.len(), 2);
|
||||
assert_eq!(error_accounts.items[0].id, "acct_beta");
|
||||
assert_eq!(error_accounts.items[1].id, "acct_invalid");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -405,7 +481,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.expect("insert xAI credential fixture");
|
||||
}
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let provider = ProviderKind::new("xai").expect("xAI Provider kind");
|
||||
let complete = store
|
||||
.list_credentials(
|
||||
@@ -486,7 +562,7 @@ async fn terminal_admin_usage_chunks_large_selections_and_preserves_exact_costs(
|
||||
let mut account_ids = vec!["acct_usage_exact".to_owned()];
|
||||
account_ids.extend((0..200).map(|index| format!("missing_account_{index}")));
|
||||
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage(
|
||||
TimeRange {
|
||||
start: now - TimeDelta::hours(1),
|
||||
@@ -578,7 +654,7 @@ async fn terminal_admin_quota_window_usage_prefers_provider_total_and_falls_back
|
||||
.await
|
||||
.expect("separate provider total from fallback components");
|
||||
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage_by_windows(&[
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_quota_window".to_owned(),
|
||||
@@ -618,7 +694,7 @@ async fn terminal_admin_mutations_keep_revision_account_and_audit_atomic() {
|
||||
.insert_provider_account(account("acct_terminal_mutation", "user-terminal-mutation"))
|
||||
.await
|
||||
.expect("insert mutation account");
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "request_terminal_mutation".to_owned(),
|
||||
@@ -691,7 +767,7 @@ async fn terminal_admin_delete_removes_enabled_accounts_in_one_transaction() {
|
||||
.expect("insert enabled account");
|
||||
}
|
||||
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone())
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: vec![
|
||||
@@ -747,7 +823,6 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
updated.name = "updated import".to_owned();
|
||||
updated.provider_credentials_json = credential_json("updated-import-secret");
|
||||
updated.availability = AccountAvailability::Banned;
|
||||
updated.availability_reason = Some("upstream_account_deactivated".to_owned());
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -765,10 +840,9 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
i64,
|
||||
Option<serde_json::Value>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select name, provider_credentials_json, credential_revision, provider_quota_json,
|
||||
availability, availability_reason
|
||||
availability
|
||||
from provider_accounts where id = 'acct_admin_upsert'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
@@ -782,7 +856,6 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
"credential replacement must clear stale quota"
|
||||
);
|
||||
assert_eq!(row.4, "banned");
|
||||
assert_eq!(row.5.as_deref(), Some("upstream_account_deactivated"));
|
||||
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
@@ -819,8 +892,6 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -942,8 +1013,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
};
|
||||
|
||||
let mut cooldown_account = account("acct_admin_b", "user-admin-b");
|
||||
cooldown_account.availability = AccountAvailability::Cooldown;
|
||||
cooldown_account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(10));
|
||||
cooldown_account.availability = AccountAvailability::Ready;
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -953,24 +1023,21 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.await
|
||||
.expect("import provider accounts");
|
||||
assert_eq!(imported.config_revision.get(), 2);
|
||||
let ready: (bool, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
|
||||
"select enabled, availability, cooldown_until from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load ready imported account");
|
||||
assert_eq!(ready, (true, "ready".to_owned(), None));
|
||||
let cooldown: (bool, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
|
||||
"select enabled, availability, cooldown_until from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_b")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load cooldown imported account");
|
||||
let ready: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load ready imported account");
|
||||
assert_eq!(ready, (true, "ready".to_owned()));
|
||||
let cooldown: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
.bind("acct_admin_b")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load cooldown imported account");
|
||||
assert!(cooldown.0);
|
||||
assert_eq!(cooldown.1, "cooldown");
|
||||
assert!(cooldown.2.is_some());
|
||||
assert_eq!(cooldown.1, "ready");
|
||||
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
@@ -1017,7 +1084,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = 'expired', availability_reason = 'credential_expired'
|
||||
set availability = 'expired'
|
||||
where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1040,15 +1107,8 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.expect("rotate provider account");
|
||||
assert_eq!(rotation.config_revision.get(), 3);
|
||||
assert_eq!(rotation.credential_revision.get(), 2);
|
||||
let restored: (
|
||||
String,
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select availability, availability_reason, cooldown_until,
|
||||
upstream_user_id, upstream_account_id
|
||||
let restored: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select availability, upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1059,8 +1119,6 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
restored,
|
||||
(
|
||||
"ready".to_owned(),
|
||||
None,
|
||||
None,
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)
|
||||
@@ -1158,7 +1216,6 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
};
|
||||
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
|
||||
exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
exhausted.availability_reason = Some("quota_exhausted".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -1187,20 +1244,13 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
.await
|
||||
.expect("rotate exhausted account credential");
|
||||
|
||||
let availability: (String, Option<String>) = sqlx::query_as(
|
||||
"select availability, availability_reason from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(
|
||||
availability,
|
||||
(
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
let availability: String =
|
||||
sqlx::query_scalar("select availability from provider_accounts where id = $1")
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(availability, "quota_exhausted".to_owned());
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1221,32 +1271,24 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("upstream_free_quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI free quota exhaustion");
|
||||
|
||||
let current: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI quota state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("upstream_free_quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1266,38 +1308,26 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
|
||||
.expect("insert xAI account");
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
let cooldown_until = observed_at
|
||||
.checked_add(Duration::from_secs(24 * 60 * 60))
|
||||
.expect("cooldown deadline");
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(cooldown_until),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI resettable usage limit");
|
||||
|
||||
let current: (String, String, Option<String>, bool) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason, cooldown_until is not null
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI usage limit state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"cooldown".to_owned(),
|
||||
Some("usage_limit_exhausted".to_owned()),
|
||||
true,
|
||||
),
|
||||
);
|
||||
assert_eq!(current, ("xai".to_owned(), "ready".to_owned()));
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1315,7 +1345,6 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
|
||||
disabled.enabled = false;
|
||||
disabled.availability = AccountAvailability::Expired;
|
||||
disabled.availability_reason = Some("credential_expired".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -1331,11 +1360,10 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1355,9 +1383,8 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
.await
|
||||
.expect("refresh disabled account credential");
|
||||
|
||||
let current: (bool, String, Option<String>, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, availability_reason,
|
||||
provider_credentials_json, credential_revision
|
||||
let current: (bool, String, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, provider_credentials_json, credential_revision
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
@@ -1366,9 +1393,8 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
.expect("load disabled account after refresh writes");
|
||||
assert!(!current.0);
|
||||
assert_eq!(current.1, "expired");
|
||||
assert_eq!(current.2.as_deref(), Some("credential_expired"));
|
||||
assert_eq!(current.3["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.4, 2);
|
||||
assert_eq!(current.2["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.3, 2);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1389,8 +1415,6 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
@@ -1517,31 +1541,15 @@ fn provider_credentials_are_redacted_from_debug() {
|
||||
assert!(!format!("{object:?}").contains(secret));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_observation_requires_expiry() {
|
||||
let observation = ProviderAccountObservation {
|
||||
account_id: "account-1".to_owned(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
provider_quota_json: None,
|
||||
availability_observed_at: Utc::now(),
|
||||
quota_observed_at: None,
|
||||
};
|
||||
assert!(observation.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
|
||||
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
|
||||
imported.availability = AccountAvailability::QuotaExhausted;
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_cooldown_until_for_ready_runtime_facts() {
|
||||
let mut imported = account("acct_invalid_cooldown", "user-invalid-cooldown");
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_err());
|
||||
fn imported_ready_account_passes_validation() {
|
||||
let imported = account("acct_ready_import", "user-ready-import");
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
@@ -6,19 +6,9 @@ use gateway_store::postgres::{
|
||||
PgObservabilityRepository, PgOpsEventRepository, UsageRecordFilter, UsageRecordQuery,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::TestDatabase;
|
||||
|
||||
type ProviderAccountSnapshotRow = (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
);
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_snapshots_should_survive_account_deletion() {
|
||||
let Some(database) = TestDatabase::create("snapshot_request_delete").await else {
|
||||
@@ -49,7 +39,7 @@ async fn request_snapshots_should_survive_account_deletion() {
|
||||
.await
|
||||
.expect("delete provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
@@ -164,7 +154,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_ab")
|
||||
.await
|
||||
@@ -286,7 +276,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
@@ -386,7 +376,7 @@ async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -455,7 +445,7 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
||||
.expect("insert legacy request without live account");
|
||||
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -521,7 +511,7 @@ async fn failure_diagnostics_should_only_include_errored_requests() {
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -576,7 +566,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
|
||||
.await
|
||||
.expect("set request b latency");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -643,7 +633,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
.await
|
||||
.expect("rename provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
@@ -664,175 +654,6 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_should_backfill_snapshots_for_existing_accounts_and_survive_deletion() {
|
||||
let Some(database_url) = crate::support::test_env("CPR_TEST_DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let schema = format!("cpr_store_snapshot_backfill_{}", Uuid::new_v4().simple());
|
||||
let admin = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect test PostgreSQL");
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!("create schema \"{schema}\"")))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("create backfill schema");
|
||||
let search_path = schema.clone();
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.after_connect(move |connection, _metadata| {
|
||||
let search_path = search_path.clone();
|
||||
Box::pin(async move {
|
||||
sqlx::query("select set_config('search_path', $1, false)")
|
||||
.bind(search_path)
|
||||
.execute(connection)
|
||||
.await?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect backfill pool");
|
||||
let started_at = Utc::now();
|
||||
{
|
||||
let mut migration = pool.begin().await.expect("begin terminal migration");
|
||||
sqlx::raw_sql(include_str!("../../../../migrations/0001_initial.sql"))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply initial migration");
|
||||
migration.commit().await.expect("commit initial migration");
|
||||
}
|
||||
seed_account(
|
||||
&pool,
|
||||
"acct_backfill",
|
||||
"Backfill",
|
||||
Some("backfill@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, requested_model_id,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
upstream_model_id, upstream_transport, attempt_count, upstream_send_state,
|
||||
started_at, deadline_at
|
||||
) values (
|
||||
'req_backfill', 'key_backfill', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'coding', 'openai', 'acct_backfill', 'acct_backfill',
|
||||
'upstream-backfill', 'http_sse', 1, 'sent', $1, $1 + interval '30 seconds'
|
||||
)",
|
||||
)
|
||||
.bind(started_at)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("insert pre-migration request");
|
||||
sqlx::query(
|
||||
"insert into ops_events (
|
||||
id, level, component, operation,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
failure_kind, message, occurrence_count, created_at
|
||||
) values (
|
||||
'ops_backfill', 'warning', 'account_probe', 'connection_test',
|
||||
'openai', 'acct_backfill', 'acct_backfill',
|
||||
'auth_failed', 'backfill probe', 1, $1
|
||||
)",
|
||||
)
|
||||
.bind(started_at)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("insert pre-migration ops event");
|
||||
{
|
||||
let mut migration = pool.begin().await.expect("begin snapshot migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0002_snapshot_provider_account_identity.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply snapshot migration");
|
||||
migration.commit().await.expect("commit snapshot migration");
|
||||
}
|
||||
let (
|
||||
request_name,
|
||||
request_email,
|
||||
request_authentication_kind,
|
||||
event_name,
|
||||
event_email,
|
||||
event_authentication_kind,
|
||||
): ProviderAccountSnapshotRow = sqlx::query_as(
|
||||
"select
|
||||
(select provider_account_name_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_email_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_authentication_kind_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_name_snapshot from ops_events where id = 'ops_backfill'),
|
||||
(select provider_account_email_snapshot from ops_events where id = 'ops_backfill'),
|
||||
(select provider_account_authentication_kind_snapshot from ops_events where id = 'ops_backfill')",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("load backfilled snapshots");
|
||||
assert_eq!(
|
||||
(
|
||||
request_name.as_deref(),
|
||||
request_email.as_deref(),
|
||||
request_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
(
|
||||
event_name.as_deref(),
|
||||
event_email.as_deref(),
|
||||
event_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
sqlx::query("delete from provider_accounts where id = 'acct_backfill'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("delete backfilled account");
|
||||
let request_snapshot: (Option<String>, Option<String>, Option<String>) = sqlx::query_as(
|
||||
"select provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot
|
||||
from model_requests where id = 'req_backfill'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("load request snapshot after deletion");
|
||||
assert_eq!(
|
||||
(
|
||||
request_snapshot.0.as_deref(),
|
||||
request_snapshot.1.as_deref(),
|
||||
request_snapshot.2.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
|
||||
drop(pool);
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"drop schema \"{schema}\" cascade"
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("drop backfill schema");
|
||||
admin.close().await;
|
||||
}
|
||||
|
||||
fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
|
||||
NewModelRequest {
|
||||
id: id.to_owned(),
|
||||
|
||||
@@ -1,27 +1,13 @@
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration as StdDuration, SystemTime},
|
||||
};
|
||||
use std::time::{Duration as StdDuration, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaObservation, QuotaWriteOutcome,
|
||||
},
|
||||
error::{StoreError as CoreStoreError, StoreErrorKind},
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
use gateway_store::{
|
||||
Revision, StoreBackend, StoreError, StoreResult,
|
||||
redis::{
|
||||
CooldownCachingProviderAccountStore, CredentialCooldown, CredentialCooldownRepository,
|
||||
RedisCredentialCooldownRepository,
|
||||
},
|
||||
Revision,
|
||||
redis::{CredentialCooldown, CredentialCooldownRepository, RedisCredentialCooldownRepository},
|
||||
};
|
||||
use redis::aio::ConnectionManager;
|
||||
use uuid::Uuid;
|
||||
@@ -36,409 +22,6 @@ fn credential_cooldown_is_revision_fenced() {
|
||||
assert_eq!(cooldown.credential_revision.get(), 2);
|
||||
}
|
||||
|
||||
struct CooldownTestAccountStore {
|
||||
accounts: Vec<ProviderAccount>,
|
||||
calls: Arc<Mutex<Vec<&'static str>>>,
|
||||
fail_state_write: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProviderAccountStore for CooldownTestAccountStore {
|
||||
async fn create_account(&self, _account: NewProviderAccount) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not create accounts")
|
||||
}
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
Ok(self
|
||||
.accounts
|
||||
.iter()
|
||||
.find(|candidate| candidate.id() == account)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
Ok(self.accounts.clone())
|
||||
}
|
||||
|
||||
async fn list_for_provider(
|
||||
&self,
|
||||
_provider: &ProviderKind,
|
||||
) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not list one provider")
|
||||
}
|
||||
|
||||
async fn load_credential(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
_expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
_update: CredentialCasUpdate,
|
||||
) -> Result<CredentialCasOutcome, CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
Ok(CredentialCasOutcome::Updated(
|
||||
CredentialRevision::new(8).expect("positive revision"),
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_quotas(
|
||||
&self,
|
||||
_accounts: &[ProviderAccountId],
|
||||
) -> Result<Vec<QuotaObservation>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not read quota")
|
||||
}
|
||||
|
||||
async fn compare_and_swap_quota(
|
||||
&self,
|
||||
_observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not write quota")
|
||||
}
|
||||
|
||||
async fn apply_state_change(&self, _change: AccountStateChange) -> Result<(), CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
if self.fail_state_write {
|
||||
Err(CoreStoreError::new(StoreErrorKind::Unavailable))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_account(&self, _update: ProviderAccountUpdate) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not update account metadata")
|
||||
}
|
||||
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
_enabled: bool,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
unreachable!("cooldown tests do not change account switches")
|
||||
}
|
||||
|
||||
async fn delete_account(&self, _account: &ProviderAccountId) -> Result<(), CoreStoreError> {
|
||||
self.calls.lock().expect("calls lock").push("postgres");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct CooldownTestCache {
|
||||
calls: Arc<Mutex<Vec<&'static str>>>,
|
||||
cached: Mutex<Vec<CredentialCooldown>>,
|
||||
invalidated: Mutex<Vec<(String, Revision)>>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CredentialCooldownRepository for CooldownTestCache {
|
||||
async fn cache_credential_cooldown(&self, cooldown: &CredentialCooldown) -> StoreResult<bool> {
|
||||
self.calls.lock().expect("calls lock").push("redis");
|
||||
if self.fail {
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::Redis,
|
||||
message: "test cache unavailable".to_owned(),
|
||||
});
|
||||
}
|
||||
self.cached
|
||||
.lock()
|
||||
.expect("cached lock")
|
||||
.push(cooldown.clone());
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn read_credential_cooldown(
|
||||
&self,
|
||||
_provider_account_id: &str,
|
||||
) -> StoreResult<Option<CredentialCooldown>> {
|
||||
unreachable!("write-through adapter does not consume cooldown cache")
|
||||
}
|
||||
|
||||
async fn invalidate_credential_cooldown(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
through_revision: Revision,
|
||||
) -> StoreResult<bool> {
|
||||
self.calls.lock().expect("calls lock").push("redis");
|
||||
if self.fail {
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::Redis,
|
||||
message: "test cache unavailable".to_owned(),
|
||||
});
|
||||
}
|
||||
self.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.push((provider_account_id.to_owned(), through_revision));
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn composition_fixture(
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
composition_fixture_with_accounts(
|
||||
vec![ready_test_account("acct_cooldown_composition")],
|
||||
fail_state_write,
|
||||
fail_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn composition_fixture_with_accounts(
|
||||
accounts: Vec<ProviderAccount>,
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts,
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write,
|
||||
});
|
||||
let cache = Arc::new(CooldownTestCache {
|
||||
calls: Arc::clone(&calls),
|
||||
cached: Mutex::new(Vec::new()),
|
||||
invalidated: Mutex::new(Vec::new()),
|
||||
fail: fail_cache,
|
||||
});
|
||||
let store = CooldownCachingProviderAccountStore::new(
|
||||
authoritative,
|
||||
Arc::clone(&cache) as Arc<dyn CredentialCooldownRepository>,
|
||||
);
|
||||
(store, cache, calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_writes_cache_only_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("persist cooldown");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
assert_eq!(cache.cached.lock().expect("cached lock").len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_does_not_cache_failed_postgres_state() {
|
||||
let (store, cache, calls) = composition_fixture(true, false);
|
||||
|
||||
assert!(
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_composition_keeps_postgres_terminal_when_redis_fails() {
|
||||
let (store, _cache, calls) = composition_fixture(false, true);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("Redis is only a cache");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_state_write_does_not_change_runtime_cooldown() {
|
||||
let disabled = test_provider_account("acct_cooldown_composition").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
);
|
||||
let (store, cache, calls) = composition_fixture_with_accounts(vec![disabled], false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
assert!(
|
||||
cache
|
||||
.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_state_invalidates_same_revision_cooldown_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
let mut change = cooldown_state_change(60);
|
||||
change.availability = AccountAvailability::Ready;
|
||||
change.reason = None;
|
||||
change.cooldown_until = None;
|
||||
|
||||
store
|
||||
.apply_state_change(change)
|
||||
.await
|
||||
.expect("persist ready state");
|
||||
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0].1,
|
||||
Revision::new(7).expect("positive revision")
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_rotation_invalidates_cooldown_through_new_revision() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
|
||||
let outcome = store
|
||||
.compare_and_swap_credential(credential_update())
|
||||
.await
|
||||
.expect("rotate credential");
|
||||
|
||||
assert_eq!(
|
||||
outcome,
|
||||
CredentialCasOutcome::Updated(CredentialRevision::new(8).expect("positive revision"))
|
||||
);
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0].1,
|
||||
Revision::new(8).expect("positive revision")
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_delete_invalidates_all_stale_cooldown_revisions() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
let account_id = ProviderAccountId::new("acct_cooldown_delete").expect("valid account ID");
|
||||
|
||||
store
|
||||
.delete_account(&account_id)
|
||||
.await
|
||||
.expect("delete account");
|
||||
|
||||
assert_eq!(
|
||||
cache.invalidated.lock().expect("invalidated lock")[0],
|
||||
(
|
||||
account_id.to_string(),
|
||||
Revision::new(u64::MAX).expect("positive revision")
|
||||
)
|
||||
);
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_startup_hydrates_only_active_postgres_cooldowns() {
|
||||
let now = SystemTime::now();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts: vec![
|
||||
cooldown_test_account("acct_hydrate_active", now + StdDuration::from_secs(60)),
|
||||
cooldown_test_account("acct_hydrate_expired", now - StdDuration::from_secs(1)),
|
||||
ready_test_account("acct_hydrate_ready"),
|
||||
],
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write: false,
|
||||
});
|
||||
let cache = Arc::new(CooldownTestCache {
|
||||
calls,
|
||||
cached: Mutex::new(Vec::new()),
|
||||
invalidated: Mutex::new(Vec::new()),
|
||||
fail: false,
|
||||
});
|
||||
let store = CooldownCachingProviderAccountStore::new(
|
||||
authoritative,
|
||||
Arc::clone(&cache) as Arc<dyn CredentialCooldownRepository>,
|
||||
);
|
||||
|
||||
assert_eq!(store.hydrate(now).await, 1);
|
||||
assert_eq!(
|
||||
cache.cached.lock().expect("cached lock")[0].provider_account_id,
|
||||
"acct_hydrate_active"
|
||||
);
|
||||
}
|
||||
|
||||
fn cooldown_state_change(seconds: u64) -> AccountStateChange {
|
||||
AccountStateChange {
|
||||
account_id: ProviderAccountId::new("acct_cooldown_composition").expect("valid account ID"),
|
||||
expected_revision: CredentialRevision::new(7).expect("positive revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: Some(SystemTime::now() + StdDuration::from_secs(seconds)),
|
||||
observed_at: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn credential_update() -> CredentialCasUpdate {
|
||||
let account_id = ProviderAccountId::new("acct_cooldown_rotation").expect("valid account ID");
|
||||
CredentialCasUpdate::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(7).expect("positive revision"),
|
||||
ProviderAccountUpdate {
|
||||
account_id,
|
||||
name: "Rotated cooldown account".to_owned(),
|
||||
email: None,
|
||||
plan_type: None,
|
||||
},
|
||||
PlaintextCredential::new(serde_json::Map::new()),
|
||||
false,
|
||||
Some(SystemTime::now() + StdDuration::from_secs(3_600)),
|
||||
None,
|
||||
)
|
||||
.expect("valid credential update")
|
||||
}
|
||||
|
||||
fn cooldown_test_account(account_id: &str, cooldown_until: SystemTime) -> ProviderAccount {
|
||||
test_provider_account(account_id).with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::Cooldown,
|
||||
Some(cooldown_until),
|
||||
)
|
||||
}
|
||||
|
||||
fn ready_test_account(account_id: &str) -> ProviderAccount {
|
||||
test_provider_account(account_id).with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
}
|
||||
|
||||
fn test_provider_account(account_id: &str) -> ProviderAccount {
|
||||
ProviderAccount::new(
|
||||
ProviderAccountId::new(account_id).expect("valid account ID"),
|
||||
ProviderKind::new("openai").expect("valid provider kind"),
|
||||
"Cooldown test".to_owned(),
|
||||
"upstream-user".to_owned(),
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(7).expect("positive revision"),
|
||||
Some(SystemTime::now() + StdDuration::from_secs(3_600)),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_cooldown_round_trips_without_raw_account_id_in_key() {
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
|
||||
@@ -475,7 +475,14 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
};
|
||||
Ok(project_quota(snapshot, &account))
|
||||
let mut quota = project_quota(snapshot, &account);
|
||||
quota.rate_limited_until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok(quota)
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -579,8 +586,6 @@ fn prepared_create(
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability,
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: observed_at,
|
||||
})
|
||||
}
|
||||
@@ -682,11 +687,7 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
|
||||
account.upstream_account_id.clone(),
|
||||
account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(
|
||||
account.enabled,
|
||||
account.availability,
|
||||
account.cooldown_until.map(SystemTime::from),
|
||||
)
|
||||
.with_runtime_state(account.enabled, account.availability)
|
||||
.with_refresh_schedule(
|
||||
account.has_refresh_token,
|
||||
account.next_refresh_at.map(SystemTime::from),
|
||||
@@ -745,6 +746,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -753,9 +756,12 @@ fn project_quota(
|
||||
snapshot: Option<CodexAccountQuotaSnapshot>,
|
||||
account: &ProviderAccount,
|
||||
) -> ProviderQuota {
|
||||
let mut quota = snapshot.map_or_else(empty_quota, project_quota_snapshot);
|
||||
// 展示前先滚动已过期窗口,避免 reset 后的旧窗口值悬挂。
|
||||
let mut quota = snapshot
|
||||
.map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now())))
|
||||
.unwrap_or_else(empty_quota);
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
force_confirmed_exhaustion_projection(&mut quota, account.cooldown_until());
|
||||
force_confirmed_exhaustion_projection(&mut quota);
|
||||
}
|
||||
quota
|
||||
}
|
||||
@@ -773,7 +779,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
"exhausted".to_owned(),
|
||||
Value::Bool(snapshot.fact().exhausted()),
|
||||
);
|
||||
let windows = snapshot
|
||||
let windows: Vec<ProviderQuotaWindow> = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.map(|window| {
|
||||
@@ -790,6 +796,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
key: window.key().to_owned(),
|
||||
group: quota_group(window.kind()).to_owned(),
|
||||
label: codex_quota_window_label(
|
||||
window.limit_name(),
|
||||
window.kind(),
|
||||
window.source(),
|
||||
window.window_seconds(),
|
||||
@@ -798,27 +805,35 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
window_seconds: window.window_seconds(),
|
||||
used_percent: window.used_percent(),
|
||||
reset_at: window.reset_at(),
|
||||
limit_reached: window.limit_reached(),
|
||||
local_usage: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(data))),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// 快照级 limit_reached 只看滚动后的窗口触顶:顶层标记是观测事实,不能
|
||||
// 在窗口全部过期后继续维持限流。
|
||||
let limit_reached = quota_windows_limit_reached(&windows);
|
||||
ProviderQuota {
|
||||
observed_at: Some(DateTime::<Utc>::from(snapshot.observed_at())),
|
||||
refresh_token_expires_at: None,
|
||||
windows,
|
||||
limit_reached,
|
||||
rate_limited_until: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
provider_data,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn force_confirmed_exhaustion_projection(
|
||||
quota: &mut ProviderQuota,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) {
|
||||
// 真实 429 对“当前已耗尽”的证明强于可能延迟结算的 usage 文档。这里只覆盖
|
||||
// Admin 展示投影,原始 Provider JSON 仍作为后续判断窗口是否真正恢复的基线。
|
||||
fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
|
||||
windows.iter().any(|window| window.limit_reached)
|
||||
}
|
||||
|
||||
fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) {
|
||||
// 402 已确认耗尽:只覆盖 Admin 展示投影,原始 Provider JSON 仍作为
|
||||
// 后续判断窗口是否真正恢复的基线。
|
||||
quota.limit_reached = true;
|
||||
quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
Map::from_iter([
|
||||
(
|
||||
@@ -828,7 +843,6 @@ fn force_confirmed_exhaustion_projection(
|
||||
("exhausted".to_owned(), Value::Bool(true)),
|
||||
]),
|
||||
)));
|
||||
let reset_at = cooldown_until.map(DateTime::<Utc>::from);
|
||||
if quota.windows.is_empty() {
|
||||
quota.windows.push(ProviderQuotaWindow {
|
||||
key: "confirmed-quota-exhaustion".to_owned(),
|
||||
@@ -837,42 +851,28 @@ fn force_confirmed_exhaustion_projection(
|
||||
source: None,
|
||||
window_seconds: None,
|
||||
used_percent: Some(100.0),
|
||||
reset_at,
|
||||
reset_at: None,
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let matched_reset = reset_at.and_then(|reset_at| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, window)| {
|
||||
let distance = window.reset_at.map(|window_reset| {
|
||||
(window_reset.timestamp() - reset_at.timestamp()).unsigned_abs()
|
||||
})?;
|
||||
Some((index, distance))
|
||||
})
|
||||
.min_by_key(|(_, distance)| *distance)
|
||||
.map(|(index, _)| index)
|
||||
});
|
||||
let index = matched_reset
|
||||
.or_else(|| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, left), (_, right)| {
|
||||
left.used_percent
|
||||
.unwrap_or(-1.0)
|
||||
.total_cmp(&right.used_percent.unwrap_or(-1.0))
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
// 选已用比例最高的窗口强制 100%,保留其自身 reset_at。
|
||||
let index = quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, left), (_, right)| {
|
||||
left.used_percent
|
||||
.unwrap_or(-1.0)
|
||||
.total_cmp(&right.used_percent.unwrap_or(-1.0))
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
quota.windows[index].used_percent = Some(100.0);
|
||||
quota.windows[index].limit_reached = true;
|
||||
}
|
||||
|
||||
const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str {
|
||||
@@ -891,11 +891,17 @@ const fn quota_role(role: CodexQuotaWindowRole) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// 窗口展示名,优先使用上游 `limit_name`(原文,`_` 转空格);
|
||||
/// 无 limit_name 时按窗口时长猜(±5% 容差,汉化输出),猜不中兜底「额度」。
|
||||
fn codex_quota_window_label(
|
||||
limit_name: Option<&str>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
source: &str,
|
||||
window_seconds: Option<u64>,
|
||||
) -> String {
|
||||
if let Some(limit_name) = limit_name {
|
||||
return limit_name.replace('_', " ");
|
||||
}
|
||||
let base = match kind {
|
||||
CodexQuotaWindowKind::Monthly => "月限额".to_owned(),
|
||||
CodexQuotaWindowKind::Weekly => "周限额".to_owned(),
|
||||
@@ -908,12 +914,9 @@ fn codex_quota_window_label(
|
||||
}
|
||||
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds),
|
||||
};
|
||||
if matches!(source, "core" | "codex" | "spend_control" | "monthly_limit") {
|
||||
if matches!(source, "core" | "codex" | "code_review" | "spend_control") {
|
||||
return base;
|
||||
}
|
||||
if is_codex_review_limit(source) {
|
||||
return format!("代码审查 · {base}");
|
||||
}
|
||||
format!("{source} · {base}")
|
||||
}
|
||||
|
||||
@@ -930,15 +933,6 @@ fn custom_quota_window_label(window_seconds: Option<u64>) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_codex_review_limit(value: &str) -> bool {
|
||||
let normalized = value.trim().to_ascii_lowercase().replace(['-', ' '], "_");
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"review" | "code_review" | "codex_review" | "codex_code_review"
|
||||
) || normalized.contains("code_review")
|
||||
|| normalized.contains("codex_review")
|
||||
}
|
||||
|
||||
/// 将 Provider-owned PKCE/OIDC 状态保存到 Store 提供的 Redis 原子端口。
|
||||
pub(crate) struct OpenAiOAuthPendingStore {
|
||||
port: Arc<dyn OAuthPendingFlowPort>,
|
||||
|
||||
@@ -295,7 +295,7 @@ impl Default for CodexWireProfileConfig {
|
||||
codex_version: "0.145.0".to_owned(),
|
||||
desktop_version: "26.721.31836".to_owned(),
|
||||
desktop_build: "5828".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
|
||||
@@ -409,7 +409,7 @@ impl CodexCredentialAdmin {
|
||||
Some(input.verified_account.chatgpt_account_id),
|
||||
input.verified_account.plan_type,
|
||||
)
|
||||
.with_runtime_state(input.enabled, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(input.enabled, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(
|
||||
input.secret.refresh_token.is_some(),
|
||||
optional_time(input.next_refresh_at),
|
||||
@@ -806,10 +806,7 @@ impl CodexCredentialAdminService {
|
||||
)) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
account.account =
|
||||
account
|
||||
.account
|
||||
.with_runtime_state(enabled, availability, None);
|
||||
account.account = account.account.with_runtime_state(enabled, availability);
|
||||
account
|
||||
}
|
||||
};
|
||||
@@ -933,7 +930,7 @@ impl CodexCredentialAdminService {
|
||||
Some(upstream_account_id),
|
||||
candidate.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(enabled, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(enabled, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(false, None);
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
@@ -1000,9 +997,7 @@ fn cpr_status(account: &ProviderAccount) -> &'static str {
|
||||
AccountAvailability::QuotaExhausted => "quota_exhausted",
|
||||
AccountAvailability::Expired | AccountAvailability::Invalid => "expired",
|
||||
AccountAvailability::Banned => "banned",
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Cooldown => "active",
|
||||
AccountAvailability::Unknown | AccountAvailability::Ready => "active",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -783,7 +783,6 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool
|
||||
&& match account.availability() {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::QuotaExhausted => true,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
||||
//! 402 恢复证据与 availability 转换。
|
||||
//!
|
||||
//! `QuotaExhausted -> Ready` 只接受同一 credential revision 的真实成功响应、
|
||||
//! 新鲜快照显式 `allowed=true` 且无 exhaustion、或已到期旧 reset 配合未耗尽新快照。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::engine::credential::AccountAvailability;
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::snapshot::CodexQuotaFact;
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
/// 402 恢复可接受的证据强度。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum QuotaRecoveryEvidence {
|
||||
/// 一次真实成功推理响应(最强证据)。
|
||||
SuccessfulResponse,
|
||||
/// 周期 worker 刷新出的新快照(需旧 reset 已到期佐证)。
|
||||
RefreshedSnapshot,
|
||||
}
|
||||
|
||||
/// 只处理 `QuotaExhausted` 的恢复——429/触顶不再进入该状态,由 quota 数据
|
||||
/// 驱动调度排除(限流不改变账号可用性)。`Invalid/Expired/Banned` 等终态由 selector 的
|
||||
/// 成功响应恢复路径处理。
|
||||
///
|
||||
/// `previous_reset_at` 是写入新快照前读取的旧确认 reset;`RefreshedSnapshot`
|
||||
/// 恢复要求它已到期(窗口真实滚动后新快照的 reset 指向下一个未来周期,
|
||||
/// 不能用它判定旧耗尽窗口是否已结束)。
|
||||
pub(crate) fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
fact: CodexQuotaFact,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
recovery_evidence: QuotaRecoveryEvidence,
|
||||
) -> Option<AccountAvailability> {
|
||||
if current != AccountAvailability::QuotaExhausted || fact.exhausted() {
|
||||
return None;
|
||||
}
|
||||
// 新鲜快照显式 `allowed=true` 且无 exhaustion,直接恢复。
|
||||
if fact.explicit_allowed() {
|
||||
return Some(AccountAvailability::Ready);
|
||||
}
|
||||
// 一次真实成功响应是最强证据;周期刷新需要旧确认 reset 已到期
|
||||
// (used_percent 是可滞后、会取整的观测值,其回落不能覆盖仍未到期的旧 reset)。
|
||||
match recovery_evidence {
|
||||
QuotaRecoveryEvidence::SuccessfulResponse => Some(AccountAvailability::Ready),
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot
|
||||
if previous_reset_at.is_some_and(|reset_at| reset_at <= now) =>
|
||||
{
|
||||
Some(AccountAvailability::Ready)
|
||||
}
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
|
||||
let CodexClientError::Upstream { status, body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) {
|
||||
return Some(AccountAvailability::Banned);
|
||||
}
|
||||
match *status {
|
||||
// 402:真额度耗尽 → QuotaExhausted(worker 按 reset_at 自动恢复)。
|
||||
StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted),
|
||||
// 429/503:临时限流,不写 availability(限流不改变账号可用性),由 quota 数据驱动。
|
||||
StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE => None,
|
||||
// quota/usage 接口只能产生额度域状态;鉴权拒绝不具备判定 RT
|
||||
// 永久失效的证据,终态只能由 OAuth refresh/身份校验状态机写入。
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(body: &str) -> bool {
|
||||
serde_json::from_str::<Value>(body)
|
||||
.ok()
|
||||
.is_some_and(|value| {
|
||||
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,645 @@
|
||||
//! 快照/窗口解析、聚合、O(1) 滚动与调度信号。
|
||||
//!
|
||||
//! Provider raw quota JSON 的唯一解析者:产出 [`CodexAccountQuotaSnapshot`]
|
||||
//! 供服务编排层消费;`limit_reached` 是快照级事实,不能只从子窗口反推。
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaObservation,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CodexQuotaFact {
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
exhausted: bool,
|
||||
/// 上游显式 `allowed=true`(恢复证据:新鲜快照明确放行)。
|
||||
explicit_allowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CodexQuotaWindowKind {
|
||||
ShortTerm,
|
||||
Weekly,
|
||||
Monthly,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CodexQuotaWindowRole {
|
||||
Primary,
|
||||
Secondary,
|
||||
Monthly,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CodexQuotaWindow {
|
||||
key: String,
|
||||
source: String,
|
||||
/// 上游提供的限流名称(`metered_feature`/`limit_name`/`limitId`);显示优先用它。
|
||||
limit_name: Option<String>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
role: CodexQuotaWindowRole,
|
||||
window_seconds: Option<u64>,
|
||||
used_percent: Option<f64>,
|
||||
reset_at: Option<DateTime<Utc>>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl CodexQuotaWindow {
|
||||
#[must_use]
|
||||
pub fn key(&self) -> &str {
|
||||
&self.key
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn source(&self) -> &str {
|
||||
&self.source
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn limit_name(&self) -> Option<&str> {
|
||||
self.limit_name.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> CodexQuotaWindowKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn role(&self) -> CodexQuotaWindowRole {
|
||||
self.role
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn window_seconds(&self) -> Option<u64> {
|
||||
self.window_seconds
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn used_percent(&self) -> Option<f64> {
|
||||
self.used_percent
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reset_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.reset_at
|
||||
}
|
||||
|
||||
/// 该窗口是否已触顶(`limit_reached` 或 `used_percent >= 100`)。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(&self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CodexAccountQuotaSnapshot {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
fact: CodexQuotaFact,
|
||||
windows: Vec<CodexQuotaWindow>,
|
||||
}
|
||||
|
||||
impl CodexAccountQuotaSnapshot {
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn observed_at(&self) -> SystemTime {
|
||||
self.observed_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn fact(&self) -> CodexQuotaFact {
|
||||
self.fact
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn windows(&self) -> &[CodexQuotaWindow] {
|
||||
&self.windows
|
||||
}
|
||||
|
||||
/// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、
|
||||
/// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。
|
||||
/// 只作用于展示/调度投影,不写回持久化 raw JSON。
|
||||
///
|
||||
/// O(1) 计算:`next_reset = reset + (floor((now - reset) / window) + 1) * window`,
|
||||
/// 不做逐周期线性循环。
|
||||
#[must_use]
|
||||
pub fn roll_expired_windows(mut self, now: SystemTime) -> Self {
|
||||
for window in &mut self.windows {
|
||||
let Some(reset_at) = window.reset_at else {
|
||||
continue;
|
||||
};
|
||||
let reset = SystemTime::from(reset_at);
|
||||
if reset > now {
|
||||
continue;
|
||||
}
|
||||
window.used_percent = Some(0.0);
|
||||
window.limit_reached = false;
|
||||
let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) else {
|
||||
// 没有合法窗口时长:清除已过期的 reset,等待下一次权威观测。
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
let elapsed = now
|
||||
.duration_since(reset)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let periods = elapsed / window_seconds + 1;
|
||||
let Some(after) =
|
||||
reset.checked_add(Duration::from_secs(periods.saturating_mul(window_seconds)))
|
||||
else {
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
window.reset_at = Some(DateTime::<Utc>::from(after));
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexQuotaFact {
|
||||
#[must_use]
|
||||
pub const fn remaining_percent(&self) -> Option<u8> {
|
||||
self.remaining_percent
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn resets_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.resets_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn exhausted(&self) -> bool {
|
||||
self.exhausted
|
||||
}
|
||||
|
||||
/// 上游是否显式 `allowed=true`(新鲜快照的恢复证据)。
|
||||
#[must_use]
|
||||
pub const fn explicit_allowed(&self) -> bool {
|
||||
self.explicit_allowed
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let mut aggregate = QuotaAggregate::default();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
aggregate.observe_rate_limit(limit.rate_limit)?;
|
||||
}
|
||||
// spend_control 只提供 exhaustion 信号(`reached`),不生成窗口、
|
||||
// 不参与剩余量聚合(`individual_limit` 等字段无窗口语义)。
|
||||
if let Some(spend_control) = object.get("spend_control") {
|
||||
aggregate.observe_exhaustion_object(spend_control, "reached")?;
|
||||
}
|
||||
if let Some(credits) = object.get("credits") {
|
||||
aggregate.observe_exhaustion_object(credits, "overage_limit_reached")?;
|
||||
}
|
||||
if !aggregate.recognized {
|
||||
return Err(CodexCredentialQuotaError::InvalidCredentialData);
|
||||
}
|
||||
Ok(CodexQuotaFact {
|
||||
remaining_percent: aggregate.remaining_percent,
|
||||
resets_at: aggregate.resets_at,
|
||||
exhausted: aggregate.exhausted,
|
||||
explicit_allowed: aggregate.explicit_allowed,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn quota_snapshot_from_observation(
|
||||
observation: &QuotaObservation,
|
||||
) -> Option<CodexAccountQuotaSnapshot> {
|
||||
let observed_at = observation.observed_at?;
|
||||
let quota = observation.quota.as_ref()?;
|
||||
parse_account_quota_snapshot(
|
||||
observation.account_id.clone(),
|
||||
observation.expected_revision,
|
||||
observed_at,
|
||||
&Value::Object(quota.expose_to_provider().clone()),
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// 调度投影缓存 TTL:`min(常规 TTL, 最近 active reset - now)`。
|
||||
/// reset 到期时缓存自然失效,不固定多等一个常规 TTL 周期。
|
||||
pub(crate) fn quota_projection_ttl(snapshot: &CodexAccountQuotaSnapshot) -> Option<Duration> {
|
||||
let age = SystemTime::now()
|
||||
.duration_since(snapshot.observed_at())
|
||||
.unwrap_or(Duration::ZERO);
|
||||
let normal_ttl = QUOTA_SCHEDULING_TTL.checked_sub(age)?;
|
||||
let now = SystemTime::now();
|
||||
let reset_ttl = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter_map(|window| window.reset_at())
|
||||
.map(SystemTime::from)
|
||||
.filter(|reset| *reset > now)
|
||||
.map(|reset| reset.duration_since(now).unwrap_or(Duration::ZERO))
|
||||
.min();
|
||||
let ttl = match reset_ttl {
|
||||
Some(reset_ttl) if reset_ttl < normal_ttl => reset_ttl,
|
||||
_ => normal_ttl,
|
||||
};
|
||||
(!ttl.is_zero()).then_some(ttl)
|
||||
}
|
||||
|
||||
/// 从快照的滚动窗口派生中立调度事实(`limit_reached` = 任一窗口触顶)。
|
||||
pub(crate) fn scheduling_signals_from_snapshot(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let rolled = snapshot.clone().roll_expired_windows(SystemTime::now());
|
||||
quota_scheduling_signals(&rolled)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_scheduling_signals(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let now = SystemTime::now();
|
||||
let mut limit_reached = false;
|
||||
let mut remaining_rank: Option<u64> = None;
|
||||
let mut reset_at: Option<SystemTime> = None;
|
||||
for window in &snapshot.windows {
|
||||
limit_reached |= window.limit_reached();
|
||||
if let Some(used) = window
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
{
|
||||
let rank = ((100.0 - used) * 100.0).round() as u64;
|
||||
remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank)));
|
||||
}
|
||||
if let Some(reset) = window
|
||||
.reset_at()
|
||||
.map(SystemTime::from)
|
||||
.filter(|reset| *reset > now)
|
||||
{
|
||||
reset_at = Some(reset_at.map_or(reset, |current| current.min(reset)));
|
||||
}
|
||||
}
|
||||
// 快照级 limit_reached 是调度排除的充分条件:即使没有任何窗口带
|
||||
// percent/reset(如顶层限流标记),也必须产出信号;不能返回 None 丢失排除。
|
||||
if limit_reached || remaining_rank.is_some() || reset_at.is_some() {
|
||||
Some(AccountQuotaSignals::new(
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_account_quota_snapshot(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
usage: &Value,
|
||||
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
|
||||
let fact = parse_codex_quota_usage(usage)?;
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let mut windows = Vec::new();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
parse_rate_limit_windows(
|
||||
&limit.source,
|
||||
limit.limit_name.as_deref(),
|
||||
limit.rate_limit,
|
||||
&mut windows,
|
||||
)?;
|
||||
}
|
||||
windows.sort_by(|left, right| left.key.cmp(&right.key));
|
||||
Ok(CodexAccountQuotaSnapshot {
|
||||
account_id,
|
||||
credential_revision,
|
||||
observed_at,
|
||||
fact,
|
||||
windows,
|
||||
})
|
||||
}
|
||||
|
||||
/// 一个规范化后的限流桶:source 是调度/展示分组键,limit_name 是上游显示名。
|
||||
struct CanonicalRateLimit<'a> {
|
||||
source: String,
|
||||
limit_name: Option<String>,
|
||||
rate_limit: &'a Value,
|
||||
}
|
||||
|
||||
fn canonical_rate_limits(
|
||||
object: &Map<String, Value>,
|
||||
) -> Result<Vec<CanonicalRateLimit<'_>>, CodexCredentialQuotaError> {
|
||||
let mut limits = Vec::new();
|
||||
if let Some(rate_limit) = object.get("rate_limit") {
|
||||
limits.push(CanonicalRateLimit {
|
||||
source: "core".to_owned(),
|
||||
limit_name: None,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
if let Some(rate_limit) = object
|
||||
.get("code_review_rate_limit")
|
||||
.filter(|value| !value.is_null())
|
||||
{
|
||||
limits.push(CanonicalRateLimit {
|
||||
source: "code_review".to_owned(),
|
||||
limit_name: Some("code_review".to_owned()),
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
let Some(additional) = object
|
||||
.get("additional_rate_limits")
|
||||
.filter(|value| !value.is_null())
|
||||
else {
|
||||
return Ok(limits);
|
||||
};
|
||||
for (index, value) in additional
|
||||
.as_array()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let item = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let (source, limit_name) = canonical_rate_limit_source(item, index);
|
||||
if let Some(rate_limit) = item.get("rate_limit") {
|
||||
// additional 里的 codex 桶覆盖顶层 rate_limit(同源替换)。
|
||||
if let Some(existing) = limits.iter_mut().find(|existing| existing.source == source) {
|
||||
existing.rate_limit = rate_limit;
|
||||
existing.limit_name = limit_name;
|
||||
} else {
|
||||
limits.push(CanonicalRateLimit {
|
||||
source,
|
||||
limit_name,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(limits)
|
||||
}
|
||||
|
||||
fn canonical_rate_limit_source(
|
||||
item: &Map<String, Value>,
|
||||
index: usize,
|
||||
) -> (String, Option<String>) {
|
||||
let source = item
|
||||
.get("metered_feature")
|
||||
.or_else(|| item.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("additional-{index}"));
|
||||
let limit_name = item
|
||||
.get("limit_name")
|
||||
.or_else(|| item.get("metered_feature"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_owned);
|
||||
if source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex" {
|
||||
("core".to_owned(), limit_name)
|
||||
} else {
|
||||
(source, limit_name)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_rate_limit_windows(
|
||||
source: &str,
|
||||
limit_name: Option<&str>,
|
||||
value: &Value,
|
||||
output: &mut Vec<CodexQuotaWindow>,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 顶层限流标记是快照级事实:任一窗口都要继承所属
|
||||
// rate_limit 的 `limit_reached=true` / `allowed=false`。
|
||||
let top_level_reached = optional_bool(object, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
for (name, role) in [
|
||||
("primary_window", CodexQuotaWindowRole::Primary),
|
||||
("secondary_window", CodexQuotaWindowRole::Secondary),
|
||||
] {
|
||||
let Some(window) = object.get(name) else {
|
||||
continue;
|
||||
};
|
||||
if window.is_null() {
|
||||
continue;
|
||||
}
|
||||
let window = window
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let window_seconds = optional_positive_u64(window, "limit_window_seconds")?
|
||||
.or(optional_positive_u64(window, "window_seconds")?)
|
||||
.or(optional_positive_u64(window, "window_minutes")?
|
||||
.and_then(|minutes| minutes.checked_mul(60)));
|
||||
let used_percent = window
|
||||
.get("used_percent")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.map(|value| value.clamp(0.0, 100.0))
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let reset_at = window
|
||||
.get("reset_at")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_i64()
|
||||
.filter(|value| *value > 0)
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value, 0))
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let limit_reached = top_level_reached
|
||||
|| window
|
||||
.get("limit_reached")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
|| used_percent.is_some_and(|used| used >= 100.0);
|
||||
let kind = quota_window_kind(window_seconds);
|
||||
output.push(CodexQuotaWindow {
|
||||
key: format!("{}-{}", quota_key(source), quota_role_name(role, kind)),
|
||||
source: source.to_owned(),
|
||||
limit_name: limit_name.map(str::to_owned),
|
||||
kind,
|
||||
role,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn optional_positive_u64(
|
||||
object: &Map<String, Value>,
|
||||
key: &str,
|
||||
) -> Result<Option<u64>, CodexCredentialQuotaError> {
|
||||
object
|
||||
.get(key)
|
||||
.map(|value| {
|
||||
value
|
||||
.as_u64()
|
||||
.filter(|value| *value > 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
const fn quota_window_kind(seconds: Option<u64>) -> CodexQuotaWindowKind {
|
||||
match seconds {
|
||||
Some(value) if value >= 17_100 && value <= 18_900 => CodexQuotaWindowKind::ShortTerm,
|
||||
Some(value) if value >= 574_560 && value <= 635_040 => CodexQuotaWindowKind::Weekly,
|
||||
Some(value) if value >= 2_462_400 && value <= 2_721_600 => CodexQuotaWindowKind::Monthly,
|
||||
_ => CodexQuotaWindowKind::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_key(value: &str) -> String {
|
||||
let mut key = value
|
||||
.chars()
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() {
|
||||
character.to_ascii_lowercase()
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
while key.contains("--") {
|
||||
key = key.replace("--", "-");
|
||||
}
|
||||
let key = key.trim_matches('-');
|
||||
if key.is_empty() {
|
||||
"quota".to_owned()
|
||||
} else {
|
||||
key.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind) -> &'static str {
|
||||
match kind {
|
||||
CodexQuotaWindowKind::ShortTerm => "five-hour",
|
||||
CodexQuotaWindowKind::Weekly => "weekly",
|
||||
CodexQuotaWindowKind::Monthly => "monthly",
|
||||
CodexQuotaWindowKind::Other => match role {
|
||||
CodexQuotaWindowRole::Primary => "primary",
|
||||
CodexQuotaWindowRole::Secondary => "secondary",
|
||||
CodexQuotaWindowRole::Monthly => "monthly",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct QuotaAggregate {
|
||||
recognized: bool,
|
||||
exhausted: bool,
|
||||
explicit_allowed: bool,
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl QuotaAggregate {
|
||||
fn observe_rate_limit(&mut self, value: &Value) -> Result<(), CodexCredentialQuotaError> {
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
|
||||
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
self.explicit_allowed |= optional_bool(object, "allowed")?.is_some_and(|allowed| allowed);
|
||||
for key in ["primary_window", "secondary_window"] {
|
||||
if let Some(window) = object.get(key) {
|
||||
self.observe_window(window)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_window(&mut self, value: &Value) -> Result<(), CodexCredentialQuotaError> {
|
||||
if value.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
if let Some(used) = object.get("used_percent") {
|
||||
let used = used
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 子窗口触顶(used>=100)是快照级 exhaustion 事实。
|
||||
self.exhausted |= used >= 100.0;
|
||||
let remaining = (100.0 - used).clamp(0.0, 100.0).round() as u8;
|
||||
self.remaining_percent = Some(
|
||||
self.remaining_percent
|
||||
.map_or(remaining, |current| current.min(remaining)),
|
||||
);
|
||||
}
|
||||
if let Some(reset) = object.get("reset_at") {
|
||||
let seconds = reset
|
||||
.as_i64()
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let reset = DateTime::<Utc>::from_timestamp(seconds, 0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.resets_at = Some(self.resets_at.map_or(reset, |current| current.min(reset)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_exhaustion_object(
|
||||
&mut self,
|
||||
value: &Value,
|
||||
key: &str,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
if value.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, key)?.unwrap_or(false);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_bool(
|
||||
object: &Map<String, Value>,
|
||||
key: &str,
|
||||
) -> Result<Option<bool>, CodexCredentialQuotaError> {
|
||||
object
|
||||
.get(key)
|
||||
.map(|value| {
|
||||
value
|
||||
.as_bool()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
@@ -192,7 +192,6 @@ impl CodexCredentialRefreshService {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
"invalid_grant",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
.await
|
||||
@@ -201,7 +200,6 @@ impl CodexCredentialRefreshService {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Banned,
|
||||
"account_banned",
|
||||
CodexCredentialRefreshOutcome::Banned { account_id },
|
||||
)
|
||||
.await
|
||||
@@ -330,7 +328,6 @@ impl CodexCredentialRefreshService {
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Invalid,
|
||||
"refreshed_identity_rejected",
|
||||
CodexCredentialRefreshOutcome::Invalidated {
|
||||
account_id: due.account.id().to_string(),
|
||||
},
|
||||
@@ -433,7 +430,6 @@ impl CodexCredentialRefreshService {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
reason: &'static str,
|
||||
outcome: CodexCredentialRefreshOutcome,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
match self.repository.load_runtime_credential(account).await {
|
||||
@@ -447,13 +443,7 @@ impl CodexCredentialRefreshService {
|
||||
}
|
||||
match self
|
||||
.repository
|
||||
.apply_state(
|
||||
account,
|
||||
availability,
|
||||
Some(reason.to_owned()),
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.apply_state(account, availability, SystemTime::now())
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(outcome),
|
||||
|
||||
@@ -233,9 +233,19 @@ impl CodexCredentialRepository {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
reason: Option<String>,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
self.apply_state_with_message(account, availability, observed_at, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 带上游错误描述的状态写入;`message` 为 `Some` 时持久化最近错误,恢复 `Ready` 时清空。
|
||||
pub async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
@@ -245,9 +255,8 @@ impl CodexCredentialRepository {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
message,
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
|
||||
@@ -32,8 +32,6 @@ use super::types::{
|
||||
RuntimeCodexCookie,
|
||||
};
|
||||
|
||||
const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const CLOUDFLARE_PATH_BLOCK_COOLDOWN: Duration = Duration::from_secs(30);
|
||||
const CLOUDFLARE_RECOVERY_STALE_AFTER: Duration = Duration::from_secs(60 * 60);
|
||||
const CLOUDFLARE_CHALLENGE_BACKOFF: [Duration; 4] = [
|
||||
Duration::from_secs(10),
|
||||
@@ -84,7 +82,6 @@ struct RiskRecoveryState {
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum CookieRecovery {
|
||||
None,
|
||||
ExpireAt(SystemTime),
|
||||
Clear,
|
||||
}
|
||||
@@ -297,6 +294,24 @@ impl CodexCredentialSelector {
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
|
||||
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
|
||||
let accounts = if diagnostic {
|
||||
accounts
|
||||
} else {
|
||||
let mut retained = Vec::with_capacity(accounts.len());
|
||||
for account in accounts {
|
||||
let cooling = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
if cooling.is_none() {
|
||||
retained.push(account);
|
||||
}
|
||||
}
|
||||
retained
|
||||
};
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -321,6 +336,7 @@ impl CodexCredentialSelector {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
})
|
||||
@@ -329,9 +345,16 @@ impl CodexCredentialSelector {
|
||||
AccountCandidate { account, signals }
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// 旧版开关只影响“候选资格”。保留原始账号供 revision-fenced 凭据加载,
|
||||
// 用影子候选承载放宽后的资格,避免伪造已持久化账号的运行状态。
|
||||
let quota_exhausted_candidates = (!self.skip_exhausted).then(|| {
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
@@ -340,8 +363,8 @@ impl CodexCredentialSelector {
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let scheduling_candidates = quota_exhausted_candidates.as_deref().unwrap_or(&candidates);
|
||||
};
|
||||
let scheduling_candidates = scheduling_candidates.as_slice();
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
@@ -680,120 +703,78 @@ impl CodexCredentialSelector {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
failure: CodexAccountFailure,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let now = SystemTime::now();
|
||||
let (availability, reason, cooldown_until, cookie_recovery) = match failure {
|
||||
CodexAccountFailure::CredentialExpired => (
|
||||
AccountAvailability::Expired,
|
||||
Some("credential_expired".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::IdentityVerificationRequired => (
|
||||
AccountAvailability::Invalid,
|
||||
Some("identity_verification_required".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::Banned => (
|
||||
AccountAvailability::Banned,
|
||||
Some("account_banned".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::QuotaExhausted => (
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::UsageLimitExhausted { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(
|
||||
account,
|
||||
retry_after,
|
||||
"usage_limit_exhausted",
|
||||
now,
|
||||
)
|
||||
.await;
|
||||
match failure {
|
||||
CodexAccountFailure::CredentialExpired => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::IdentityVerificationRequired => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::Banned => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
|
||||
.await
|
||||
}
|
||||
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
|
||||
CodexAccountFailure::QuotaExhausted
|
||||
| CodexAccountFailure::UsageLimitExhausted { .. } => {
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// 429:临时限流只写 quota,availability 不改变。
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(account, retry_after, "rate_limited", now)
|
||||
.await;
|
||||
self.quota
|
||||
.apply_rate_limit_429(account, retry_after, now)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
// Cloudflare 挑战:内存退避表(记录风险计数),不写 availability。
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after } => {
|
||||
let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after);
|
||||
let cooldown_until = now.checked_add(delay);
|
||||
(
|
||||
AccountAvailability::Cooldown,
|
||||
Some("cloudflare_challenge".to_owned()),
|
||||
cooldown_until,
|
||||
cooldown_until.map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt),
|
||||
)
|
||||
let recovery = now
|
||||
.checked_add(delay)
|
||||
.map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt);
|
||||
self.apply_cookie_recovery(account, recovery).await?;
|
||||
Ok(())
|
||||
}
|
||||
// Cloudflare 路径被封:连续超阈值才置 Invalid;否则内存退避。
|
||||
CodexAccountFailure::CloudflarePathBlocked => {
|
||||
let blocked = self.record_cloudflare_path_block(account.id(), now);
|
||||
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
|
||||
(
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::Invalid,
|
||||
Some("cloudflare_path_blocked".to_owned()),
|
||||
None,
|
||||
CookieRecovery::Clear,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
AccountAvailability::Cooldown,
|
||||
Some("cloudflare_path_blocked".to_owned()),
|
||||
now.checked_add(CLOUDFLARE_PATH_BLOCK_COOLDOWN),
|
||||
CookieRecovery::Clear,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
self.apply_cookie_recovery(account, CookieRecovery::Clear)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
self.repository
|
||||
.apply_state(account, availability, reason, cooldown_until, now)
|
||||
.await?;
|
||||
self.apply_cookie_recovery(account, cookie_recovery).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_resettable_limit_failure(
|
||||
async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
reason: &'static str,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let current = self.current_account(account.id()).await?;
|
||||
if current.revision() != account.revision() {
|
||||
return Err(CredentialRepositoryError::RevisionConflict.into());
|
||||
}
|
||||
let cooldown_until =
|
||||
observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN));
|
||||
let cooldown_until = match (current.availability(), current.cooldown_until()) {
|
||||
(AccountAvailability::QuotaExhausted, Some(existing)) => {
|
||||
cooldown_until.map_or(Some(existing), |until| Some(existing.max(until)))
|
||||
}
|
||||
_ => cooldown_until,
|
||||
};
|
||||
let reason = if current.availability() == AccountAvailability::QuotaExhausted
|
||||
&& current.cooldown_until().is_none()
|
||||
{
|
||||
Some("quota_exhausted".to_owned())
|
||||
} else {
|
||||
Some(reason.to_owned())
|
||||
};
|
||||
// 一次真实 429 证明上游已经接受当前凭据,可纠正旧的认证/封禁终态。
|
||||
// resettable 用量限制沿用 v2 的 quota lock 语义,并保留/延长复核时间。
|
||||
self.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.apply_state_with_message(account, availability, observed_at, message)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -857,7 +838,6 @@ impl CodexCredentialSelector {
|
||||
|| !matches!(
|
||||
current.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
@@ -867,13 +847,7 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.apply_state(¤t, AccountAvailability::Ready, SystemTime::now())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -985,15 +959,11 @@ impl CodexCredentialSelector {
|
||||
account: &ProviderAccount,
|
||||
recovery: CookieRecovery,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
if matches!(recovery, CookieRecovery::None) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut data = self.repository.load_complete_data(account).await?;
|
||||
if data.cookies().is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
match recovery {
|
||||
CookieRecovery::None => return Ok(()),
|
||||
CookieRecovery::ExpireAt(expires_at) => {
|
||||
let expires_at = chrono::DateTime::<chrono::Utc>::from(expires_at);
|
||||
for cookie in data.cookies_mut() {
|
||||
@@ -1013,11 +983,9 @@ impl CodexCredentialSelector {
|
||||
|
||||
fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount {
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
return account.clone().with_runtime_state(
|
||||
account.enabled(),
|
||||
AccountAvailability::Ready,
|
||||
account.cooldown_until(),
|
||||
);
|
||||
return account
|
||||
.clone()
|
||||
.with_runtime_state(account.enabled(), AccountAvailability::Ready);
|
||||
}
|
||||
account.clone()
|
||||
}
|
||||
@@ -1032,7 +1000,6 @@ fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> Af
|
||||
}
|
||||
match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountAvailability::Cooldown => AffinityEscapeReason::Cooldown,
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
|
||||
@@ -111,6 +111,7 @@ pub async fn initialize(
|
||||
http.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
ports.cooldowns(),
|
||||
));
|
||||
let quota_skip_exhausted = config.quota_skip_exhausted();
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
|
||||
@@ -1697,6 +1697,8 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,随 availability 状态一并持久化。
|
||||
error_message: Option<String>,
|
||||
cyber_policy_failure: bool,
|
||||
set_cookie_headers: Vec<String>,
|
||||
rate_limit_headers: Vec<(String, String)>,
|
||||
@@ -1709,6 +1711,7 @@ impl MappedProviderFailure {
|
||||
Self {
|
||||
error,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
set_cookie_headers: Vec::new(),
|
||||
rate_limit_headers: Vec::new(),
|
||||
@@ -1882,7 +1885,7 @@ async fn apply_failure(
|
||||
.await;
|
||||
if let Err(error) = context
|
||||
.selector
|
||||
.record_failure(account, account_failure)
|
||||
.record_failure(account, account_failure, failure.error_message.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -2162,6 +2165,7 @@ fn map_upstream_failure(
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
account_failure: account_failure(category, failure.retry_after_seconds),
|
||||
error_message: failure.client_message,
|
||||
cyber_policy_failure,
|
||||
set_cookie_headers: failure.set_cookie_headers,
|
||||
rate_limit_headers: failure.rate_limit_headers,
|
||||
@@ -2533,7 +2537,6 @@ impl ScheduledTask for OpenAiQuotaTask {
|
||||
updated = summary.updated,
|
||||
exhausted = summary.exhausted,
|
||||
banned = summary.banned,
|
||||
cooldown = summary.cooldown,
|
||||
transient = summary.transient,
|
||||
stale = summary.stale,
|
||||
"OpenAI quota cycle contained operational failures"
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::{
|
||||
WebSocketPoolConnectLease, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
|
||||
WebSocketPoolLease,
|
||||
},
|
||||
pump::{PumpKeepalive, PumpedWebSocket},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket},
|
||||
};
|
||||
|
||||
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
|
||||
@@ -203,10 +203,12 @@ async fn prepare_unpooled_websocket(
|
||||
decision_started_at: Instant,
|
||||
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
|
||||
let permit = acquire_breaker_permit(breaker, origin_key)?;
|
||||
let context = pump_log_context_from_connection(request.connection());
|
||||
let connected = connect_with_budget(
|
||||
request.connection(),
|
||||
PumpKeepalive::disabled(),
|
||||
fast_path_budget,
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
let (connection, connect_elapsed) = finish_breaker_attempt(permit, connected)?;
|
||||
@@ -302,6 +304,10 @@ fn start_pooled_websocket_connect(
|
||||
let cancellation = connect_lease.cancellation_token();
|
||||
let fast_path_reporter = permit.fast_path_reporter();
|
||||
let keepalive = pool.keepalive();
|
||||
let context = PumpLogContext::new(
|
||||
Some(task_key.account_id().to_owned()),
|
||||
Some(task_key.conversation_id_hash()),
|
||||
);
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
pool.spawn_connect_task(async move {
|
||||
let connected = tokio::select! {
|
||||
@@ -319,7 +325,7 @@ fn start_pooled_websocket_connect(
|
||||
);
|
||||
return;
|
||||
}
|
||||
result = connect_with_budget(&connection, keepalive, None) => result,
|
||||
result = connect_with_budget(&connection, keepalive, None, context) => result,
|
||||
};
|
||||
match finish_breaker_attempt(permit, connected) {
|
||||
Ok((connection, connect_elapsed)) => {
|
||||
@@ -392,17 +398,31 @@ fn duration_millis_u64(duration: Duration) -> u64 {
|
||||
.max(1)
|
||||
}
|
||||
|
||||
/// 非池化建连的 pump 日志上下文:从业务头提取账号归属(会话 hash 只有池 key 才有)。
|
||||
fn pump_log_context_from_connection(connection: &CodexWebSocketConnection) -> PumpLogContext {
|
||||
let account_id = connection
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("chatgpt-account-id"))
|
||||
.map(|(_, value)| value.clone());
|
||||
PumpLogContext::new(account_id, None)
|
||||
}
|
||||
|
||||
async fn connect_with_budget(
|
||||
connection: &CodexWebSocketConnection,
|
||||
keepalive: PumpKeepalive,
|
||||
fast_path_budget: Option<Duration>,
|
||||
context: PumpLogContext,
|
||||
) -> Result<(PooledWebSocketConnection, Duration), CodexWebSocketExchangeError> {
|
||||
let started_at = Instant::now();
|
||||
let connected = match fast_path_budget {
|
||||
Some(budget) => timeout(budget, connect_pumped_websocket(connection, keepalive))
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
|
||||
None => connect_pumped_websocket(connection, keepalive).await,
|
||||
Some(budget) => timeout(
|
||||
budget,
|
||||
connect_pumped_websocket(connection, keepalive, context),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
|
||||
None => connect_pumped_websocket(connection, keepalive, context).await,
|
||||
}?;
|
||||
let (websocket, response) = connected;
|
||||
Ok((
|
||||
|
||||
@@ -30,7 +30,7 @@ use super::{
|
||||
error::CodexWebSocketExchangeError,
|
||||
model::{CodexWebSocketConnection, CodexWebSocketRequest, WebSocketContinuationRequirement},
|
||||
pool::CodexWebSocketConnectionMetadata,
|
||||
pump::{PumpKeepalive, PumpedWebSocket, RawWsStream},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket, RawWsStream},
|
||||
};
|
||||
|
||||
const WEBSOCKET_EXTENSIONS: &str = "permessage-deflate; client_max_window_bits";
|
||||
@@ -93,9 +93,10 @@ pub fn responses_websocket_endpoint(base_url: &str) -> String {
|
||||
pub(super) async fn connect_pumped_websocket(
|
||||
connection: &CodexWebSocketConnection,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
) -> Result<(PumpedWebSocket, WsResponse<Option<Vec<u8>>>), CodexWebSocketExchangeError> {
|
||||
let (raw, response) = connect_websocket(connection).await?;
|
||||
Ok((PumpedWebSocket::new(raw, keepalive), response))
|
||||
Ok((PumpedWebSocket::new(raw, keepalive, context), response))
|
||||
}
|
||||
|
||||
pub(super) async fn send_websocket_request(
|
||||
|
||||
@@ -36,6 +36,22 @@ pub(crate) type RawWsStream = WebSocketStream<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
const PUMP_COMMAND_BUFFER: usize = 32;
|
||||
|
||||
/// pump 日志的账号级关联上下文;空闲连接被上游重置等场景用于回溯归属。
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct PumpLogContext {
|
||||
pub(crate) account_id: Option<String>,
|
||||
pub(crate) conversation_id_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl PumpLogContext {
|
||||
pub(crate) fn new(account_id: Option<String>, conversation_id_hash: Option<String>) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
conversation_id_hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// pump 保活策略。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct PumpKeepalive {
|
||||
@@ -188,7 +204,11 @@ impl std::fmt::Debug for PumpedWebSocket {
|
||||
|
||||
impl PumpedWebSocket {
|
||||
/// 用底层流启动一个 pump 任务并返回句柄。
|
||||
pub(crate) fn new(inner: RawWsStream, keepalive: PumpKeepalive) -> Self {
|
||||
pub(crate) fn new(
|
||||
inner: RawWsStream,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
) -> Self {
|
||||
let (tx_command, rx_command) = mpsc::channel::<PumpCommand>(PUMP_COMMAND_BUFFER);
|
||||
let (tx_message, rx_message) = mpsc::channel(PUMP_MESSAGE_BUFFER);
|
||||
let connection_id = Uuid::new_v4();
|
||||
@@ -203,6 +223,7 @@ impl PumpedWebSocket {
|
||||
rx_command,
|
||||
tx_message,
|
||||
keepalive,
|
||||
context,
|
||||
closed_for_task,
|
||||
exit_reason_for_task,
|
||||
)
|
||||
@@ -273,12 +294,17 @@ impl Drop for PumpedWebSocket {
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "pump owns one flat set of channel/state handles and log context"
|
||||
)]
|
||||
async fn pump_loop(
|
||||
connection_id: Uuid,
|
||||
mut inner: RawWsStream,
|
||||
mut rx_command: mpsc::Receiver<PumpCommand>,
|
||||
tx_message: mpsc::Sender<Result<Message, tungstenite::Error>>,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
closed: Arc<AtomicBool>,
|
||||
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
|
||||
) {
|
||||
@@ -429,7 +455,7 @@ async fn pump_loop(
|
||||
*exit_reason
|
||||
.lock()
|
||||
.expect("WebSocket pump exit reason lock poisoned") = Some(reason.clone());
|
||||
log_pump_exit(connection_id, &reason, backpressure_events);
|
||||
log_pump_exit(connection_id, &context, &reason, backpressure_events);
|
||||
|
||||
if reason.should_send_close() {
|
||||
let _ = inner.send(Message::Close(None)).await;
|
||||
@@ -510,11 +536,20 @@ async fn send_keepalive_ping(
|
||||
}))
|
||||
}
|
||||
|
||||
fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_events: u64) {
|
||||
fn log_pump_exit(
|
||||
connection_id: Uuid,
|
||||
context: &PumpLogContext,
|
||||
reason: &PumpExitReason,
|
||||
backpressure_events: u64,
|
||||
) {
|
||||
let detail = reason.detail().unwrap_or_default();
|
||||
let account_id = context.account_id.as_deref().unwrap_or("unknown");
|
||||
let conversation_id_hash = context.conversation_id_hash.as_deref().unwrap_or("unknown");
|
||||
if reason.is_unexpected() {
|
||||
tracing::warn!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
@@ -523,6 +558,8 @@ fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_even
|
||||
} else if matches!(reason, PumpExitReason::UpstreamCloseFrame { .. }) {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
@@ -531,6 +568,8 @@ fn log_pump_exit(connection_id: Uuid, reason: &PumpExitReason, backpressure_even
|
||||
} else {
|
||||
tracing::debug!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
|
||||
@@ -471,6 +471,7 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -498,7 +499,8 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].label, "月限额");
|
||||
// limit_name 优先显示,不再猜成「月限额」。
|
||||
assert_eq!(monthly[0].label, "custom codex label");
|
||||
assert_eq!(monthly[0].source.as_deref(), Some("core"));
|
||||
assert_eq!(monthly[0].used_percent, Some(2.0));
|
||||
}
|
||||
@@ -536,16 +538,16 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist raw quota");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -583,11 +585,10 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -645,7 +646,6 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit(
|
||||
stale_record.next_refresh_at = None;
|
||||
stale_record.enabled = false;
|
||||
stale_record.availability = AdminAccountAvailability::Banned;
|
||||
stale_record.cooldown_until = Some(Utc::now() + chrono::Duration::minutes(5));
|
||||
let rotation_error = admin
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: stale_record,
|
||||
@@ -712,9 +712,8 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability: AdminAccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -901,6 +900,13 @@ impl ProviderCooldownPort for TestCooldown {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
struct TestRuntimePolicy;
|
||||
|
||||
@@ -194,7 +194,7 @@ fn agent_export_item(id: &str) -> (ExportManagedCodexCredential, String) {
|
||||
Some("agent-account-export".to_owned()),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(false, None);
|
||||
(
|
||||
ExportManagedCodexCredential {
|
||||
|
||||
@@ -66,7 +66,7 @@ fn agent_account(
|
||||
Some("agent-account".to_owned()),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(false, None);
|
||||
NewProviderAccount {
|
||||
account,
|
||||
|
||||
@@ -12,7 +12,9 @@ use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAffinityPort};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::OFFICIAL_CODEX_BASE_URL;
|
||||
use provider_openai::credential::{
|
||||
@@ -25,8 +27,8 @@ use secrecy::ExposeSecret;
|
||||
use url::Url;
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
};
|
||||
|
||||
fn create_account(store: &Arc<MemoryAccountStore>, id: &str, token: &str) {
|
||||
@@ -99,6 +101,7 @@ fn selector_with_affinity(
|
||||
session_affinity,
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,6 +111,7 @@ fn selector_with_runtime(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> CodexCredentialSelector {
|
||||
let profile = CodexWireProfileState::new(CodexWireProfile {
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -136,6 +140,7 @@ fn selector_with_runtime(
|
||||
http,
|
||||
OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
cooldowns,
|
||||
));
|
||||
CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
@@ -464,6 +469,7 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
|
||||
affinity,
|
||||
Arc::clone(&account_feedback),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
for _ in 0..4 {
|
||||
account_feedback.report(
|
||||
@@ -507,11 +513,10 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -550,57 +555,6 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selector_should_escape_an_affinity_account_during_cooldown() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_first", "at-first");
|
||||
create_account(&store, "acct_second", "at-second");
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: Some(SystemTime::now() + Duration::from_secs(60)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark first account cooling down");
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let key = ProviderSessionAffinityKey::try_new("cooldown-session").expect("affinity key");
|
||||
affinity
|
||||
.bind(&provider, &key, first.id(), Duration::from_secs(60))
|
||||
.await
|
||||
.expect("seed affinity");
|
||||
let selector =
|
||||
selector_with_affinity(&store, Arc::new(TestLeaseCoordinator::default()), affinity);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
let request_attempt = attempt(BTreeSet::new());
|
||||
|
||||
let selected = selector
|
||||
.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &request_url,
|
||||
attempt: &request_attempt,
|
||||
session_affinity_key: Some(&key),
|
||||
})
|
||||
.await
|
||||
.expect("select fallback account");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
selected.account_id().as_str(),
|
||||
selected.affinity_hit(),
|
||||
selected.escape_reason(),
|
||||
selected.account_switch(),
|
||||
),
|
||||
("acct_second", false, Some("cooldown"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_honors_attempt_local_account_exclusion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -716,8 +670,12 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
}),
|
||||
)
|
||||
.expect("select account");
|
||||
block_on(selector.record_failure(lease.account(), CodexAccountFailure::CredentialExpired))
|
||||
.expect("record credential expiry");
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CredentialExpired,
|
||||
None,
|
||||
))
|
||||
.expect("record credential expiry");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
@@ -728,10 +686,18 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() {
|
||||
fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
let attempt = attempt(BTreeSet::new());
|
||||
let lease =
|
||||
block_on(
|
||||
@@ -750,15 +716,22 @@ fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
|
||||
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
store.quota_json("acct_primary").is_none(),
|
||||
"429 must not synthesize quota window"
|
||||
);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown expiring in the future"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -775,17 +748,13 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("record usage-limit exhaustion");
|
||||
|
||||
let account = store.account("acct_primary").expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
}
|
||||
|
||||
@@ -795,11 +764,10 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
@@ -810,9 +778,11 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability,402 的 QuotaExhausted 保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
@@ -820,47 +790,6 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_keeps_longer_existing_quota_deadline() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let existing_deadline = SystemTime::now() + Duration::from_secs(300);
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(existing_deadline),
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
))
|
||||
.expect("record shorter rate-limit failure");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until(),
|
||||
Some(existing_deadline)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -868,13 +797,22 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
@@ -883,22 +821,27 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_rate_limit_recovers_stale_authentication_state() {
|
||||
fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Invalid,
|
||||
reason: Some("identity_verification_required".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark account invalid");
|
||||
@@ -910,15 +853,17 @@ fn authenticated_rate_limit_recovers_stale_authentication_state() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability(限流不改变账号可用性),认证终态保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
@@ -933,11 +878,10 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: stale,
|
||||
reason: Some("stale_observation".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("seed stale state");
|
||||
@@ -973,12 +917,14 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
selector.record_failure(
|
||||
&second,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
);
|
||||
first_result.expect("record first account failure");
|
||||
@@ -996,7 +942,7 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
.expect("persisted second account")
|
||||
.availability(),
|
||||
],
|
||||
[AccountAvailability::QuotaExhausted; 2]
|
||||
[AccountAvailability::Ready; 2]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1006,7 +952,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
@@ -1015,6 +961,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
@@ -1060,6 +1007,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::IdentityVerificationRequired,
|
||||
None,
|
||||
))
|
||||
.expect("record identity verification failure");
|
||||
|
||||
@@ -1082,7 +1030,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloudflare_challenge_backoff_escalates_and_success_resets_it() {
|
||||
fn cloudflare_challenge_does_not_change_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
@@ -1099,48 +1047,22 @@ fn cloudflare_challenge_backoff_escalates_and_success_resets_it() {
|
||||
)
|
||||
.expect("select account");
|
||||
|
||||
// Cloudflare 挑战只走内存退避表,不写 availability(限流不改变账号可用性)。
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record first challenge");
|
||||
let first = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("first cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(first >= Duration::from_secs(9) && first <= Duration::from_secs(10));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
))
|
||||
.expect("record second challenge");
|
||||
let second = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("second cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(second >= Duration::from_secs(29) && second <= Duration::from_secs(30));
|
||||
.expect("record challenge");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
block_on(selector.record_success(lease.account(), None));
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
))
|
||||
.expect("record reset challenge");
|
||||
let reset = store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.cooldown_until()
|
||||
.expect("reset cooldown")
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("future cooldown");
|
||||
assert!(reset >= Duration::from_secs(9) && reset <= Duration::from_secs(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1166,9 +1088,11 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
|
||||
.expect("select account");
|
||||
|
||||
for _ in 0..3 {
|
||||
block_on(
|
||||
selector.record_failure(lease.account(), CodexAccountFailure::CloudflarePathBlocked),
|
||||
)
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
}
|
||||
|
||||
@@ -1222,18 +1146,17 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() {
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record challenge");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let cooldown_until = account.cooldown_until().expect("cooldown");
|
||||
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
|
||||
assert_eq!(data.cookies().len(), 1);
|
||||
assert!(
|
||||
data.cookies()[0]
|
||||
.expires_at
|
||||
.is_some_and(|expires_at| SystemTime::from(expires_at) <= cooldown_until)
|
||||
);
|
||||
assert!(data.cookies()[0].expires_at.is_some_and(|expires_at| {
|
||||
let expires_at = SystemTime::from(expires_at);
|
||||
expires_at > SystemTime::now() && expires_at <= SystemTime::now() + Duration::from_secs(120)
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1267,8 +1190,12 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
|
||||
session_affinity_key: None,
|
||||
}))
|
||||
.expect("select revised account");
|
||||
block_on(selector.record_failure(second.account(), CodexAccountFailure::CloudflarePathBlocked))
|
||||
.expect("record path block");
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
|
||||
@@ -1304,7 +1231,7 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
|
||||
|
||||
assert_eq!(outcome.credential_revision, Some(current.revision().get()));
|
||||
assert_ne!(current.revision(), lease.account().revision());
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("record failure with current revision");
|
||||
assert_eq!(
|
||||
store
|
||||
|
||||
+115
-449
@@ -1,16 +1,23 @@
|
||||
//! quota 测试入口与共享 fixture。
|
||||
//!
|
||||
//! - [`snapshot`]:快照聚合、窗口滚动与调度信号回归。
|
||||
//! - [`recovery`]:402 恢复证据回归。
|
||||
|
||||
mod recovery;
|
||||
mod snapshot;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use futures::future::join_all;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, OpaqueProviderData, ProviderAccountStore as _,
|
||||
QuotaObservation, QuotaWriteOutcome,
|
||||
QuotaObservation,
|
||||
};
|
||||
use provider_openai::OFFICIAL_CODEX_BASE_URL;
|
||||
use provider_openai::credential::{
|
||||
CodexCredentialQuotaError, CodexCredentialQuotaService, CodexQuotaSyncSummary,
|
||||
CodexQuotaWindowKind, ImportCodexOAuthCredential, parse_codex_quota_usage,
|
||||
CodexQuotaWindowKind, ImportCodexOAuthCredential,
|
||||
};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use serde_json::json;
|
||||
@@ -19,138 +26,6 @@ use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::support::{MemoryAccountStore, profile, secret};
|
||||
|
||||
#[test]
|
||||
fn parser_extracts_dynamic_windows_without_a_fixed_database_shape() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {"used_percent": 25.2, "reset_at": 1_800_000_100},
|
||||
"secondary_window": {"used_percent": 80.4, "reset_at": 1_800_000_200}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
"limit_name": "future_dynamic_window",
|
||||
"rate_limit": {"primary_window": {"used_percent": 10}}
|
||||
}]
|
||||
}))
|
||||
.expect("valid dynamic quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(20));
|
||||
assert_eq!(
|
||||
fact.resets_at().map(|value| value.timestamp()),
|
||||
Some(1_800_000_100)
|
||||
);
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {"primary_window": {"used_percent": 10}},
|
||||
"additional_rate_limits": [{
|
||||
"rate_limit": {"allowed": false, "primary_window": {"used_percent": 100}}
|
||||
}]
|
||||
}))
|
||||
.expect("valid quota");
|
||||
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid full-percent quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(0));
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 98, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid provider-confirmed quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(2));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"credits": {
|
||||
"has_credits": false,
|
||||
"balance": 0,
|
||||
"overage_limit_reached": false,
|
||||
"future_provider_field": {"anything": true}
|
||||
}
|
||||
}))
|
||||
.expect("recognized credits object");
|
||||
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_accepts_official_null_additional_rate_limits() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"limit_window_seconds": 18_000,
|
||||
"used_percent": 12.5,
|
||||
"reset_at": 1_800_000_100
|
||||
},
|
||||
"secondary_window": {
|
||||
"limit_window_seconds": 604_800,
|
||||
"used_percent": 30.0,
|
||||
"reset_at": 1_800_000_200
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": null,
|
||||
"credits": {
|
||||
"has_credits": false,
|
||||
"balance": null,
|
||||
"overage_limit_reached": false,
|
||||
"unlimited": false
|
||||
},
|
||||
"spend_control": {
|
||||
"individual_limit": null,
|
||||
"reached": false
|
||||
}
|
||||
}))
|
||||
.expect("official null additional quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(70));
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_rejects_wrong_known_field_type_without_echoing_body() {
|
||||
let marker = "quota-secret-marker";
|
||||
let error = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {"allowed": marker}
|
||||
}))
|
||||
.expect_err("known field type must be strict");
|
||||
|
||||
assert!(!format!("{error:?} {error}").contains(marker));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_rejects_unrecognized_top_level_object() {
|
||||
assert!(parse_codex_quota_usage(&json!({"future_only": {"used": 1}})).is_err());
|
||||
}
|
||||
|
||||
fn wire_profile() -> CodexWireProfileState {
|
||||
CodexWireProfileState::new(CodexWireProfile {
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -196,6 +71,7 @@ fn quota_service_with_base_url(
|
||||
http,
|
||||
base_url,
|
||||
crate::support::agent_identity_service(store),
|
||||
Arc::new(crate::support::MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -232,133 +108,7 @@ async fn create_account_with_enabled(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_cold_scheduling_hydration_reads_quota_once() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_hydration").await;
|
||||
let account = store.account("acct_hydration").expect("created account");
|
||||
let service = quota_service(&store);
|
||||
|
||||
join_all((0..32).map(|_| service.prepare_scheduling(std::slice::from_ref(&account)))).await;
|
||||
|
||||
assert_eq!(store.quota_reads(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_provider_quota_projects_dynamic_windows_without_network_io() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_quota").await;
|
||||
let account = store.account("acct_quota").expect("created account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 37,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 18_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 42,
|
||||
"reset_at": 1_900_604_800,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": null,
|
||||
"spend_control": {
|
||||
"reached": false,
|
||||
"individual_limit": {
|
||||
"used_percent": 12,
|
||||
"reset_at": 1_902_592_000
|
||||
}
|
||||
},
|
||||
"provider_specific_root": {"opaque": [1, 2, 3]}
|
||||
});
|
||||
let outcome = store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
assert_eq!(outcome, QuotaWriteOutcome::Updated);
|
||||
|
||||
let snapshot = quota_service(&store)
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 3);
|
||||
assert_eq!(
|
||||
snapshot.windows()[0].kind(),
|
||||
CodexQuotaWindowKind::ShortTerm
|
||||
);
|
||||
assert_eq!(snapshot.windows()[0].window_seconds(), Some(18_000));
|
||||
assert_eq!(snapshot.windows()[1].kind(), CodexQuotaWindowKind::Weekly);
|
||||
assert_eq!(snapshot.windows()[2].kind(), CodexQuotaWindowKind::Monthly);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_canonical_codex_limit").await;
|
||||
let account = store
|
||||
.account("acct_canonical_codex_limit")
|
||||
.expect("created account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 91,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 2,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
let outcome = store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
assert_eq!(outcome, QuotaWriteOutcome::Updated);
|
||||
|
||||
let snapshot = quota_service(&store)
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
let monthly = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| window.kind() == CodexQuotaWindowKind::Monthly)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(98));
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].source(), "core");
|
||||
assert_eq!(monthly[0].used_percent(), Some(2.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_rate_limit_headers_update_quota_and_account_state_with_revision_fence() {
|
||||
async fn passive_rate_limit_headers_update_quota_with_revision_fence() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_quota").await;
|
||||
let account = store.account("acct_passive_quota").expect("account");
|
||||
@@ -384,7 +134,8 @@ async fn passive_rate_limit_headers_update_quota_and_account_state_with_revision
|
||||
let current = store
|
||||
.account("acct_passive_quota")
|
||||
.expect("current account");
|
||||
assert_eq!(current.availability(), AccountAvailability::QuotaExhausted);
|
||||
// 被动头限流只写 quota,不改变 availability(限流不改变账号可用性)。
|
||||
assert_eq!(current.availability(), AccountAvailability::Ready);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
@@ -434,132 +185,46 @@ async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() {
|
||||
assert_eq!(
|
||||
(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
snapshot.fact().remaining_percent(),
|
||||
snapshot.fact().exhausted(),
|
||||
),
|
||||
(AccountAvailability::Ready, None, Some(0), false)
|
||||
// used=100 是快照级 exhaustion,但 availability 保持 Ready
|
||||
// (限流不改变账号可用性)。
|
||||
(AccountAvailability::Ready, Some(0), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_reset() {
|
||||
async fn passive_headers_should_materialize_false_when_quota_is_not_exhausted() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_quota_state";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
reason: Some("stale_expired_state".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed stale expired state");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
create_account(&store, "acct_passive_quota_available").await;
|
||||
let account = store
|
||||
.account("acct_passive_quota_available")
|
||||
.expect("account");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "codex".to_owned()),
|
||||
("x-codex-primary-used-percent".to_owned(), "31".to_owned()),
|
||||
("x-codex-allowed".to_owned(), "true".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
|
||||
let exhausted = service
|
||||
.refresh_account(account.id())
|
||||
let observations = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("refresh exhausted quota");
|
||||
|
||||
assert!(exhausted.fact().exhausted());
|
||||
.expect("read quota observation");
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("exhausted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let stale = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh contradictory full quota");
|
||||
|
||||
assert!(!stale.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after contradictory quota")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"an unchanged usage window cannot override a confirmed quota failure"
|
||||
);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let unchanged_reset = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh lower usage quota");
|
||||
|
||||
assert!(!unchanged_reset.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after lower usage")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"a lower used_percent before the recorded reset is not quota recovery"
|
||||
observations
|
||||
.first()
|
||||
.and_then(|observation| observation.limit_reached),
|
||||
Some(false)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -571,11 +236,10 @@ async fn manual_quota_refresh_preserves_disabled_account_state() {
|
||||
let account = store.account(account_id).expect("created disabled account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
reason: Some("disabled_expired_state".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -626,11 +290,10 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential(
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -732,6 +395,7 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
@@ -782,7 +446,6 @@ async fn synchronize_without_accounts_is_a_noop_before_network_io() {
|
||||
assert_eq!(summary.updated, 0);
|
||||
assert_eq!(summary.exhausted, 0);
|
||||
assert_eq!(summary.banned, 0);
|
||||
assert_eq!(summary.cooldown, 0);
|
||||
assert_eq!(summary.transient, 0);
|
||||
assert_eq!(summary.stale, 0);
|
||||
}
|
||||
@@ -811,6 +474,7 @@ async fn periodic_quota_synchronization_does_not_scan_stale_ready_accounts() {
|
||||
)),
|
||||
// 正常账号即使缓存已旧,也只等待真实请求被动同步,不能被定时任务主动拉取。
|
||||
observed_at: Some(SystemTime::UNIX_EPOCH),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist stale quota");
|
||||
@@ -824,7 +488,7 @@ async fn periodic_quota_synchronization_does_not_scan_stale_ready_accounts() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_quota() {
|
||||
async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_stuck_cooldown").await;
|
||||
let account = store
|
||||
@@ -833,11 +497,10 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -858,6 +521,7 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist exhausted quota");
|
||||
@@ -884,19 +548,11 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q
|
||||
|
||||
service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
// 429/触顶不再写入 availability(限流不改变账号可用性),worker 只更新 quota 数据。
|
||||
let reconciled = store
|
||||
.account("acct_periodic_stuck_cooldown")
|
||||
.expect("reconciled account");
|
||||
assert_eq!(
|
||||
(reconciled.availability(), reconciled.cooldown_until()),
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some(
|
||||
SystemTime::UNIX_EPOCH
|
||||
+ Duration::from_secs(u64::try_from(reset_at).expect("positive reset")),
|
||||
),
|
||||
)
|
||||
);
|
||||
assert_eq!(reconciled.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -908,11 +564,10 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -938,15 +593,51 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
|
||||
.account("acct_periodic_exhausted")
|
||||
.expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhausted_quota() {
|
||||
async fn periodic_quota_synchronization_respects_retry_after_on_service_unavailable() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_503_retry_after").await;
|
||||
let account = store
|
||||
.account("acct_periodic_503_retry_after")
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
// 5xx 短退避重试后仍失败,Retry-After 用于写冷却,不再落入"未分类"。
|
||||
.respond_with(ResponseTemplate::new(503).insert_header("Retry-After", "120"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
let account = store
|
||||
.account("acct_periodic_503_retry_after")
|
||||
.expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_skips_ready_accounts_without_quota_reached() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_temporary_cooldown").await;
|
||||
let account = store
|
||||
@@ -954,11 +645,10 @@ async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhauste
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_add(Duration::from_secs(60)),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -975,7 +665,7 @@ async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhauste
|
||||
.account("acct_periodic_temporary_cooldown")
|
||||
.expect("cooldown account")
|
||||
.availability(),
|
||||
AccountAvailability::Cooldown
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
@@ -988,11 +678,10 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() {
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1015,11 +704,10 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1028,7 +716,8 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(1)
|
||||
// 5xx 短退避重试 1s/2s 后仍失败,最终计入 transient。
|
||||
.expect(3)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
@@ -1075,16 +764,16 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed post-failure quota snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1116,7 +805,8 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.await
|
||||
.expect("unchanged quota recheck");
|
||||
|
||||
assert_eq!(unchanged.updated, 1);
|
||||
// used=100 是快照级 exhaustion,计 exhausted 而非 updated。
|
||||
assert_eq!(unchanged.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
@@ -1130,8 +820,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
// 无 allowed/limit_reached 标记的模糊快照。
|
||||
"primary_window": {"used_percent": 99, "reset_at": reset_at}
|
||||
}
|
||||
})))
|
||||
@@ -1172,24 +861,23 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_sub(Duration::from_secs(60)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted with an elapsed reset");
|
||||
let server = MockServer::start().await;
|
||||
let next_reset = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
let elapsed_reset = Utc::now().timestamp() - 1;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 1, "reset_at": next_reset}
|
||||
"primary_window": {"used_percent": 1, "reset_at": elapsed_reset}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
@@ -1212,7 +900,6 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
.account(account_id)
|
||||
.expect("account after elapsed reset");
|
||||
assert_eq!(current.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(current.cooldown_until(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1224,11 +911,10 @@ async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() {
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1247,15 +933,12 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
|
||||
let account_id = "acct_periodic_new_reset";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let initial_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(1);
|
||||
let confirmed_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(2);
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(initial_reset),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -1282,10 +965,6 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
|
||||
);
|
||||
|
||||
let first = service.synchronize().await.expect("first reset recheck");
|
||||
let persisted_reset = store
|
||||
.account(account_id)
|
||||
.expect("account after first recheck")
|
||||
.cooldown_until();
|
||||
let second = service
|
||||
.synchronize()
|
||||
.await
|
||||
@@ -1298,20 +977,8 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
|
||||
.len();
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
first.exhausted,
|
||||
persisted_reset,
|
||||
second.exhausted,
|
||||
repeated,
|
||||
request_count,
|
||||
),
|
||||
(
|
||||
1,
|
||||
Some(confirmed_reset),
|
||||
0,
|
||||
CodexQuotaSyncSummary::default(),
|
||||
1,
|
||||
)
|
||||
(first.exhausted, second.exhausted, repeated, request_count,),
|
||||
(1, 0, CodexQuotaSyncSummary::default(), 1,)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1323,11 +990,10 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -0,0 +1,213 @@
|
||||
//! 402 恢复回归。
|
||||
//!
|
||||
//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:旧 reset 已到期 + 新快照未耗尽。
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_reset() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_quota_state";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed stale expired state");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let exhausted = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh exhausted quota");
|
||||
|
||||
assert!(exhausted.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("exhausted account")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"manual refresh never changes non-QuotaExhausted availability"
|
||||
);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let stale = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh contradictory full quota");
|
||||
|
||||
// used=100 是快照级 exhaustion。
|
||||
assert!(stale.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after contradictory quota")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"an unchanged usage window cannot override an expired credential state"
|
||||
);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let unchanged_reset = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh lower usage quota");
|
||||
|
||||
assert!(!unchanged_reset.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after lower usage")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"a lower used_percent before the recorded reset is not quota recovery for an expired credential"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_old_reset_with_fresh_allowed_snapshot_recovers_ready() {
|
||||
// 旧快照 reset_at < now,新快照 allowed=true、
|
||||
// limit_reached=false、reset_at > now;最终必须恢复 Ready。
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_cross_window_recovery";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let old_reset = Utc::now().timestamp() - 60;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": old_reset,
|
||||
"limit_window_seconds": 3600
|
||||
}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed old exhausted snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
let future_reset = Utc::now().timestamp() + 3600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 5, "reset_at": future_reset}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let recovered = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh new window quota");
|
||||
|
||||
assert!(!recovered.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after cross-window recovery")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
"old reset elapsed + fresh allowed snapshot must recover QuotaExhausted"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
//! 快照聚合、窗口滚动和调度信号回归。
|
||||
//!
|
||||
//! 覆盖 raw JSON 解析、`limit_reached` 快照级聚合与窗口投影。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use futures::future::join_all;
|
||||
use gateway_core::engine::credential::{
|
||||
OpaqueProviderData, ProviderAccountStore as _, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use provider_openai::credential::{CodexQuotaWindowKind, parse_codex_quota_usage};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{MemoryAccountStore, create_account, quota_service};
|
||||
|
||||
#[test]
|
||||
fn parser_extracts_dynamic_windows_without_a_fixed_database_shape() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {"used_percent": 25.2, "reset_at": 1_800_000_100},
|
||||
"secondary_window": {"used_percent": 80.4, "reset_at": 1_800_000_200}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
"limit_name": "future_dynamic_window",
|
||||
"rate_limit": {"primary_window": {"used_percent": 10}}
|
||||
}]
|
||||
}))
|
||||
.expect("valid dynamic quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(20));
|
||||
assert_eq!(
|
||||
fact.resets_at().map(|value| value.timestamp()),
|
||||
Some(1_800_000_100)
|
||||
);
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {"primary_window": {"used_percent": 10}},
|
||||
"additional_rate_limits": [{
|
||||
"rate_limit": {"allowed": false, "primary_window": {"used_percent": 100}}
|
||||
}]
|
||||
}))
|
||||
.expect("valid quota");
|
||||
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid full-percent quota");
|
||||
|
||||
// 任一子窗口 used>=100 是快照级 exhaustion 事实,
|
||||
// 即使顶层 allowed=true。
|
||||
assert_eq!(fact.remaining_percent(), Some(0));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 98, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid provider-confirmed quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(2));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"credits": {
|
||||
"has_credits": false,
|
||||
"balance": 0,
|
||||
"overage_limit_reached": false,
|
||||
"future_provider_field": {"anything": true}
|
||||
}
|
||||
}))
|
||||
.expect("recognized credits object");
|
||||
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_accepts_official_null_additional_rate_limits() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"limit_window_seconds": 18_000,
|
||||
"used_percent": 12.5,
|
||||
"reset_at": 1_800_000_100
|
||||
},
|
||||
"secondary_window": {
|
||||
"limit_window_seconds": 604_800,
|
||||
"used_percent": 30.0,
|
||||
"reset_at": 1_800_000_200
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": null,
|
||||
"credits": {
|
||||
"has_credits": false,
|
||||
"balance": null,
|
||||
"overage_limit_reached": false,
|
||||
"unlimited": false
|
||||
},
|
||||
"spend_control": {
|
||||
"individual_limit": null,
|
||||
"reached": false
|
||||
}
|
||||
}))
|
||||
.expect("official null additional quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(70));
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_rejects_wrong_known_field_type_without_echoing_body() {
|
||||
let marker = "quota-secret-marker";
|
||||
let error = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {"allowed": marker}
|
||||
}))
|
||||
.expect_err("known field type must be strict");
|
||||
|
||||
assert!(!format!("{error:?} {error}").contains(marker));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_rejects_unrecognized_top_level_object() {
|
||||
assert!(parse_codex_quota_usage(&json!({"future_only": {"used": 1}})).is_err());
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn concurrent_cold_scheduling_hydration_reads_quota_once() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_hydration").await;
|
||||
let account = store.account("acct_hydration").expect("created account");
|
||||
let service = quota_service(&store);
|
||||
|
||||
join_all((0..32).map(|_| service.prepare_scheduling(std::slice::from_ref(&account)))).await;
|
||||
|
||||
assert_eq!(store.quota_reads(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_provider_quota_projects_dynamic_windows_without_network_io() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_quota").await;
|
||||
let account = store.account("acct_quota").expect("created account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 37,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 18_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 42,
|
||||
"reset_at": 1_900_604_800,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": null,
|
||||
"spend_control": {
|
||||
"reached": false,
|
||||
"individual_limit": {
|
||||
"used_percent": 12,
|
||||
"reset_at": 1_902_592_000
|
||||
}
|
||||
},
|
||||
"provider_specific_root": {"opaque": [1, 2, 3]}
|
||||
});
|
||||
let outcome = store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
assert_eq!(outcome, QuotaWriteOutcome::Updated);
|
||||
|
||||
let snapshot = quota_service(&store)
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 2);
|
||||
assert_eq!(
|
||||
snapshot.windows()[0].kind(),
|
||||
CodexQuotaWindowKind::ShortTerm
|
||||
);
|
||||
assert_eq!(snapshot.windows()[0].window_seconds(), Some(18_000));
|
||||
assert_eq!(snapshot.windows()[1].kind(), CodexQuotaWindowKind::Weekly);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
|
||||
// 过去数年的 1 秒窗口:O(1) 完成,不能线性循环。
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_o1_roll").await;
|
||||
let account = store.account("acct_o1_roll").expect("account");
|
||||
let old_reset = SystemTime::now() - Duration::from_secs(5 * 365 * 24 * 3600);
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": old_reset
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("epoch")
|
||||
.as_secs(),
|
||||
"limit_window_seconds": 1
|
||||
}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
let service = quota_service(&store);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
let rolled = snapshot.roll_expired_windows(SystemTime::now());
|
||||
let window = &rolled.windows()[0];
|
||||
assert_eq!(window.used_percent(), Some(0.0));
|
||||
assert!(!window.limit_reached());
|
||||
// 下一 reset 必须落在 now 之后(O(1) 公式而非线性累加)。
|
||||
assert!(
|
||||
window
|
||||
.reset_at()
|
||||
.is_some_and(|reset| SystemTime::from(reset) > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_signal_only_limit").await;
|
||||
let account = store.account("acct_signal_only_limit").expect("account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 0}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
let service = quota_service(&store);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
assert!(snapshot.fact().exhausted());
|
||||
let signals = service.scheduling_signals(&account);
|
||||
assert!(
|
||||
signals.is_some_and(|signals| signals.limit_reached()),
|
||||
"limit_reached without percent/reset must still produce scheduling exclusion"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_canonical_codex_limit").await;
|
||||
let account = store
|
||||
.account("acct_canonical_codex_limit")
|
||||
.expect("created account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 91,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 2,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
let outcome = store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
assert_eq!(outcome, QuotaWriteOutcome::Updated);
|
||||
|
||||
let snapshot = quota_service(&store)
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
let monthly = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| window.kind() == CodexQuotaWindowKind::Monthly)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(98));
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].source(), "core");
|
||||
assert_eq!(monthly[0].used_percent(), Some(2.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn code_review_limit_projects_as_independent_window_with_limit_name() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_code_review").await;
|
||||
let account = store.account("acct_code_review").expect("account");
|
||||
let raw = json!({
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 37,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 18_000
|
||||
}
|
||||
},
|
||||
"code_review_rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 80,
|
||||
"reset_at": 1_900_604_800,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
"limit_name": "code review",
|
||||
"metered_feature": "code_review",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 55,
|
||||
"reset_at": 1_900_604_800,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}],
|
||||
"spend_control": {
|
||||
"reached": false,
|
||||
"individual_limit": {
|
||||
"used_percent": 12,
|
||||
"reset_at": 1_902_592_000
|
||||
}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
|
||||
let snapshot = quota_service(&store)
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
// spend_control 不再生成窗口(只作 exhaustion 信号),
|
||||
// 顶层 code_review_rate_limit 与 additional 的 code_review 桶同源替换后保留一个。
|
||||
let review = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| window.source() == "code_review")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(review.len(), 1);
|
||||
assert_eq!(review[0].limit_name(), Some("code review"));
|
||||
assert_eq!(review[0].used_percent(), Some(55.0));
|
||||
assert_eq!(snapshot.windows().len(), 2);
|
||||
assert!(!snapshot.fact().exhausted());
|
||||
}
|
||||
@@ -621,8 +621,6 @@ async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() {
|
||||
.apply_state(
|
||||
&deferred,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
@@ -685,18 +683,11 @@ async fn expired_access_token_refreshes_during_active_cooldown() {
|
||||
.await
|
||||
.expect("seed expired access token with deferred refresh");
|
||||
let deferred = store.account("acct_refresh").expect("deferred account");
|
||||
let cooldown_until = SystemTime::now() + Duration::from_secs(12 * 60 * 60);
|
||||
store
|
||||
.repository()
|
||||
.apply_state(
|
||||
&deferred,
|
||||
AccountAvailability::Cooldown,
|
||||
Some("rate_limited".to_owned()),
|
||||
Some(cooldown_until),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.apply_state(&deferred, AccountAvailability::Ready, SystemTime::now())
|
||||
.await
|
||||
.expect("mark account cooling down");
|
||||
.expect("mark account ready");
|
||||
let refresher = Arc::new(Refresher::new(Ok(success_tokens())));
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
@@ -724,8 +715,7 @@ async fn expired_access_token_refreshes_during_active_cooldown() {
|
||||
["rt-old-access"]
|
||||
);
|
||||
let refreshed = store.account("acct_refresh").expect("refreshed account");
|
||||
assert_eq!(refreshed.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(refreshed.cooldown_until(), Some(cooldown_until));
|
||||
assert_eq!(refreshed.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
refreshed
|
||||
.access_token_expires_at()
|
||||
|
||||
@@ -43,8 +43,9 @@ use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service_with_pool, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service_with_pool, catalog_cache, profile,
|
||||
secret,
|
||||
};
|
||||
use crate::transport::accept_codex_test_websocket;
|
||||
|
||||
@@ -136,6 +137,7 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
Arc::clone(&agent_identity),
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
));
|
||||
let account_feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -1263,12 +1265,8 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc
|
||||
vec!["response.created", "response.failed"]
|
||||
);
|
||||
let account = store.account(account_id).expect("rate-limited account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
// 429 只记录限流窗口,不改变 availability(限流不改变账号可用性)。
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
let _ = release.send(());
|
||||
server.abort();
|
||||
let _ = server.await;
|
||||
@@ -1297,6 +1295,7 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed stale passive quota");
|
||||
@@ -1358,11 +1357,6 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
let account = store.account(account_id).expect("usage-limit account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
assert_eq!(
|
||||
@@ -1799,8 +1793,6 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
.apply_state(
|
||||
&account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
@@ -1907,7 +1899,7 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers(
|
||||
.account(account_id)
|
||||
.expect("account after successful response")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
assert!(store.has_quota(account_id));
|
||||
}
|
||||
@@ -1969,16 +1961,8 @@ async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() {
|
||||
.account(account_id)
|
||||
.expect("account after HTTP SSE response");
|
||||
assert_eq!(
|
||||
(
|
||||
account.availability(),
|
||||
account.cooldown_until(),
|
||||
store.has_quota(account_id),
|
||||
),
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)),
|
||||
true,
|
||||
)
|
||||
(account.availability(), store.has_quota(account_id),),
|
||||
(AccountAvailability::Ready, true,)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,10 @@ use gateway_core::engine::credential::{
|
||||
};
|
||||
use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
|
||||
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderSessionAffinityKey,
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
|
||||
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
|
||||
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
|
||||
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
ProviderStoreError,
|
||||
};
|
||||
@@ -117,6 +118,17 @@ impl MemoryAccountStore {
|
||||
.is_some_and(|stored| stored.quota.is_some())
|
||||
}
|
||||
|
||||
pub(crate) fn quota_json(&self, id: &str) -> Option<serde_json::Value> {
|
||||
let id = ProviderAccountId::new(id).ok()?;
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("account store lock")
|
||||
.get(&id)
|
||||
.and_then(|stored| stored.quota.as_ref())
|
||||
.and_then(|observation| observation.quota.as_ref())
|
||||
.map(|data| serde_json::Value::Object(data.expose_to_provider().clone()))
|
||||
}
|
||||
|
||||
pub(crate) fn fail_provider_listing(&self) {
|
||||
self.fail_provider_listing.store(true, Ordering::SeqCst);
|
||||
}
|
||||
@@ -235,7 +247,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
revision: next,
|
||||
enabled: stored.account.enabled(),
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
access_token_expires_at,
|
||||
has_refresh_token,
|
||||
next_refresh_at,
|
||||
@@ -303,7 +314,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
enabled: stored.account.enabled(),
|
||||
availability: change.availability,
|
||||
cooldown_until: change.cooldown_until,
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -325,7 +335,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
enabled: stored.account.enabled(),
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -350,7 +359,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
enabled,
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -378,7 +386,6 @@ struct AccountRebuild {
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
@@ -407,11 +414,7 @@ fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> Provid
|
||||
current.upstream_account_id().map(str::to_owned),
|
||||
plan_type,
|
||||
)
|
||||
.with_runtime_state(
|
||||
rebuild.enabled,
|
||||
rebuild.availability,
|
||||
rebuild.cooldown_until,
|
||||
)
|
||||
.with_runtime_state(rebuild.enabled, rebuild.availability)
|
||||
.with_refresh_schedule(rebuild.has_refresh_token, rebuild.next_refresh_at)
|
||||
}
|
||||
|
||||
@@ -464,6 +467,7 @@ impl ProviderLeasePort for TestLeaseCoordinator {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -765,6 +769,93 @@ pub(crate) fn codex_account(id: &str) -> ProviderAccount {
|
||||
Some(format!("chatgpt-{id}")),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(true, Some(SystemTime::now() + Duration::from_secs(2_700)))
|
||||
}
|
||||
|
||||
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
|
||||
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct MemoryCooldownPort {
|
||||
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
|
||||
}
|
||||
|
||||
impl MemoryCooldownPort {
|
||||
#[must_use]
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
|
||||
let should_write = cooldowns.get(cooldown.account_id()).is_none_or(|current| {
|
||||
current.credential_revision() < cooldown.credential_revision()
|
||||
|| (current.credential_revision() == cooldown.credential_revision()
|
||||
&& current.until() < cooldown.until())
|
||||
});
|
||||
if should_write {
|
||||
cooldowns.insert(cooldown.account_id().clone(), cooldown);
|
||||
}
|
||||
Ok(should_write)
|
||||
})
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
Ok(self
|
||||
.cooldowns
|
||||
.lock()
|
||||
.expect("cooldown lock")
|
||||
.get(account_id)
|
||||
.cloned())
|
||||
})
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ fn wire_profile_should_generate_codex_core_user_agent() {
|
||||
codex_version: "0.144.2".to_owned(),
|
||||
desktop_version: "26.707.72221".to_owned(),
|
||||
desktop_build: "72221".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
@@ -78,7 +78,7 @@ fn wire_profile_should_generate_codex_core_user_agent() {
|
||||
|
||||
assert_eq!(
|
||||
profile.user_agent(),
|
||||
"Codex Desktop/0.144.2 (Mac OS 15.7.1; arm64) unknown"
|
||||
"Codex Desktop/0.144.2 (macOS 15.7.1; arm64) unknown"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ fn wire_profile() -> CodexWireProfile {
|
||||
codex_version: "0.144.2".to_owned(),
|
||||
desktop_version: "26.707.72221".to_owned(),
|
||||
desktop_build: "72221".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
|
||||
@@ -33,7 +33,7 @@ uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
zeroize = { version = "1.8.2", features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
p256 = { version = "0.13.2", features = ["ecdsa", "pkcs8"] }
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
tokio = { version = "1.52.3", features = ["macros", "rt"] }
|
||||
wiremock = "0.6.5"
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_core::provider_ports::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome, ProviderCooldownPort,
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
};
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
@@ -143,6 +143,7 @@ pub(crate) struct XaiAdminProvider {
|
||||
quota: Arc<GrokCredentialQuotaService>,
|
||||
catalog: Arc<GrokCredentialCatalogService>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
}
|
||||
|
||||
pub(crate) struct XaiAdminServices {
|
||||
@@ -154,6 +155,7 @@ pub(crate) struct XaiAdminServices {
|
||||
pub(crate) quota: Arc<GrokCredentialQuotaService>,
|
||||
pub(crate) catalog: Arc<GrokCredentialCatalogService>,
|
||||
pub(crate) runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
pub(crate) cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
}
|
||||
|
||||
impl XaiAdminProvider {
|
||||
@@ -178,6 +180,7 @@ impl XaiAdminProvider {
|
||||
quota: services.quota,
|
||||
catalog: services.catalog,
|
||||
runtime_policy: services.runtime_policy,
|
||||
cooldowns: services.cooldowns,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,7 +328,10 @@ impl ProviderAdmin for XaiAdminProvider {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
async fn account_unavailable(&self, _account_id: &ProviderAccountId) {}
|
||||
async fn account_unavailable(&self, account_id: &ProviderAccountId) {
|
||||
// 删除/禁用账号:清除该账号全部 account/model scoped cooldown key。
|
||||
let _ = self.cooldowns.clear_all(account_id).await;
|
||||
}
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
@@ -913,8 +919,6 @@ fn prepared_create(
|
||||
next_refresh_at: account.next_refresh_at().map(Into::into),
|
||||
enabled: account.enabled(),
|
||||
availability: account.availability(),
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(Into::into),
|
||||
availability_observed_at: observed_at,
|
||||
})
|
||||
}
|
||||
@@ -1018,11 +1022,7 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
|
||||
account.upstream_account_id.clone(),
|
||||
account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(
|
||||
account.enabled,
|
||||
account.availability,
|
||||
account.cooldown_until.map(Into::into),
|
||||
)
|
||||
.with_runtime_state(account.enabled, account.availability)
|
||||
.with_refresh_schedule(
|
||||
account.has_refresh_token,
|
||||
account.next_refresh_at.map(Into::into),
|
||||
@@ -1047,6 +1047,8 @@ fn project_quota(
|
||||
observed_at: None,
|
||||
refresh_token_expires_at,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
};
|
||||
@@ -1087,6 +1089,9 @@ fn project_quota(
|
||||
window_seconds,
|
||||
used_percent: billing.used_percent(),
|
||||
reset_at: billing.period_end().and_then(parse_utc),
|
||||
limit_reached: billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
local_usage: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(data))),
|
||||
}
|
||||
@@ -1099,6 +1104,7 @@ fn project_quota(
|
||||
window_seconds: Some(crate::GROK_FREE_ROLLING_WINDOW_SECONDS),
|
||||
used_percent: None,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: rolling_usage,
|
||||
provider_data: None,
|
||||
}
|
||||
@@ -1107,6 +1113,10 @@ fn project_quota(
|
||||
observed_at: Some(snapshot.observed_at()),
|
||||
refresh_token_expires_at,
|
||||
windows: vec![window],
|
||||
limit_reached: billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -451,6 +451,9 @@ impl GrokCredentialQuotaService {
|
||||
loaded.account.revision(),
|
||||
document,
|
||||
observed_at.into(),
|
||||
presentation
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
)
|
||||
.await
|
||||
.map_err(map_quota_repository_error)?;
|
||||
@@ -482,7 +485,6 @@ impl GrokCredentialQuotaService {
|
||||
GrokCredentialAvailability::QuotaExhausted
|
||||
)
|
||||
.then_some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
@@ -1151,7 +1153,6 @@ fn eligible_catalog_candidate(candidate: &LoadedGrokCredential) -> bool {
|
||||
&& match account.availability() {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::QuotaExhausted => true,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
@@ -1294,8 +1295,15 @@ fn quota_scheduling_signals(billing: &GrokBillingPresentation) -> Option<Account
|
||||
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
|
||||
.map(|value| SystemTime::from(value.to_utc()))
|
||||
.filter(|reset_at| *reset_at > now);
|
||||
(remaining_rank.is_some() || reset_at.is_some())
|
||||
.then(|| AccountQuotaSignals::new(reset_at, remaining_rank))
|
||||
(remaining_rank.is_some() || reset_at.is_some()).then(|| {
|
||||
AccountQuotaSignals::new(
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool {
|
||||
@@ -1343,7 +1351,6 @@ fn quota_refresh_may_update_state(before: &ProviderAccount, current: &ProviderAc
|
||||
.is_some_and(|expires_at| expires_at > SystemTime::now())
|
||||
&& current.revision() == before.revision()
|
||||
&& current.availability() == before.availability()
|
||||
&& current.cooldown_until() == before.cooldown_until()
|
||||
}
|
||||
|
||||
fn quota_refresh_availability(
|
||||
@@ -1357,15 +1364,13 @@ fn quota_refresh_availability(
|
||||
AccountAvailability::QuotaExhausted => {
|
||||
matches!(exhausted, Some(false)).then_some(GrokCredentialAvailability::Ready)
|
||||
}
|
||||
AccountAvailability::Cooldown | AccountAvailability::Unknown => {
|
||||
exhausted.map(|exhausted| {
|
||||
if exhausted {
|
||||
GrokCredentialAvailability::QuotaExhausted
|
||||
} else {
|
||||
GrokCredentialAvailability::Ready
|
||||
}
|
||||
})
|
||||
}
|
||||
AccountAvailability::Unknown => exhausted.map(|exhausted| {
|
||||
if exhausted {
|
||||
GrokCredentialAvailability::QuotaExhausted
|
||||
} else {
|
||||
GrokCredentialAvailability::Ready
|
||||
}
|
||||
}),
|
||||
AccountAvailability::Ready => {
|
||||
matches!(exhausted, Some(true)).then_some(GrokCredentialAvailability::QuotaExhausted)
|
||||
}
|
||||
|
||||
@@ -651,18 +651,11 @@ impl GrokCredentialRefreshService {
|
||||
}
|
||||
|
||||
async fn runtime_cooldown_active(&self, credential: &DueGrokCredential) -> bool {
|
||||
// runtime cooldown 是账号/model 级生命周期,与 credential revision 无关:
|
||||
// 轮换不清除、不因 revision 变化判失效(与 selector 的读取语义一致)。
|
||||
let Ok(Some(cooldown)) = self.cooldowns.read(credential.account_id()).await else {
|
||||
return false;
|
||||
};
|
||||
if cooldown.credential_revision() != credential.credential_revision() {
|
||||
if cooldown.credential_revision() < credential.credential_revision() {
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.clear(credential.account_id(), credential.credential_revision())
|
||||
.await;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
cooldown.until() > SystemTime::now()
|
||||
}
|
||||
|
||||
@@ -705,7 +698,6 @@ impl GrokCredentialRefreshService {
|
||||
expected_revision: credential.credential_revision,
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -852,8 +844,6 @@ fn account_due(
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
)
|
||||
&& (availability != AccountAvailability::Cooldown
|
||||
|| account.cooldown_until().is_some_and(|until| until <= now))
|
||||
&& account.next_refresh_at().is_some_and(|next| next <= now)
|
||||
}
|
||||
|
||||
|
||||
@@ -168,11 +168,7 @@ impl GrokCredentialAdmin {
|
||||
input.account.upstream_account_id.clone(),
|
||||
input.account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(
|
||||
input.enabled,
|
||||
input.initial_availability.into(),
|
||||
input.initial_cooldown_until.map(to_system_time),
|
||||
)
|
||||
.with_runtime_state(input.enabled, input.initial_availability.into())
|
||||
.with_refresh_schedule(true, Some(to_system_time(input.next_refresh_at)));
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
@@ -222,7 +218,6 @@ impl GrokCredentialAdmin {
|
||||
enabled: input.enabled,
|
||||
initial_availability: GrokCredentialAvailability::Ready,
|
||||
initial_availability_reason: None,
|
||||
initial_cooldown_until: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -434,16 +429,14 @@ impl GrokCredentialRepository {
|
||||
input: &UpdateGrokCredentialState,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
validate_reason(input.availability_reason.as_deref())?;
|
||||
validate_cooldown(input.availability, input.cooldown_until)?;
|
||||
self.ensure_xai_account(&input.account_id).await?;
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: input.expected_revision,
|
||||
availability: input.availability.into(),
|
||||
reason: input.availability_reason.clone(),
|
||||
cooldown_until: input.cooldown_until.map(to_system_time),
|
||||
observed_at: to_system_time(input.observed_at),
|
||||
message: input.availability_reason.clone(),
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)
|
||||
@@ -527,6 +520,7 @@ impl GrokCredentialRepository {
|
||||
expected_revision: CredentialRevision,
|
||||
document: serde_json::Map<String, Value>,
|
||||
observed_at: SystemTime,
|
||||
limit_reached: bool,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
let outcome = self
|
||||
.store
|
||||
@@ -535,6 +529,7 @@ impl GrokCredentialRepository {
|
||||
expected_revision,
|
||||
quota: Some(OpaqueProviderData::new(document)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(limit_reached),
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
@@ -708,8 +703,7 @@ fn validate_create(input: &CreateGrokCredential) -> Result<(), GrokCredentialRep
|
||||
validate_profile(&input.account)?;
|
||||
validate_refresh_schedule(input.next_refresh_at, input.account.access_token_expires_at)?;
|
||||
validate_secret(&input.secret)?;
|
||||
validate_reason(input.initial_availability_reason.as_deref())?;
|
||||
validate_cooldown(input.initial_availability, input.initial_cooldown_until)
|
||||
validate_reason(input.initial_availability_reason.as_deref())
|
||||
}
|
||||
|
||||
fn validate_profile(profile: &GrokAccountProfile) -> Result<(), GrokCredentialRepositoryError> {
|
||||
@@ -816,18 +810,6 @@ fn validate_reason(reason: Option<&str>) -> Result<(), GrokCredentialRepositoryE
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_cooldown(
|
||||
availability: GrokCredentialAvailability,
|
||||
cooldown_until: Option<DateTime<Utc>>,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
if matches!(availability, GrokCredentialAvailability::Cooldown) != cooldown_until.is_some() {
|
||||
return Err(GrokCredentialRepositoryError::InvalidInput(
|
||||
"cooldown_until",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_xai(account: &ProviderAccount) -> Result<(), GrokCredentialRepositoryError> {
|
||||
if account.provider().as_str() == XAI_PROVIDER_KIND
|
||||
&& account.authentication_kind() == XAI_AUTHENTICATION_KIND_OAUTH
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
|
||||
AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId,
|
||||
@@ -300,36 +300,26 @@ impl GrokAccountSessionSelector {
|
||||
for ((account, account_read), model_read) in
|
||||
accounts.iter().zip(account_reads).zip(model_reads)
|
||||
{
|
||||
if let Ok(Some(cooldown)) = account_read {
|
||||
if cooldown.credential_revision() != account.revision() {
|
||||
if cooldown.credential_revision() < account.revision() {
|
||||
let _ = self.cooldowns.clear(account.id(), account.revision()).await;
|
||||
}
|
||||
} else if cooldown.until() > now {
|
||||
insert_runtime_cooldown(
|
||||
&mut cooled,
|
||||
account.id(),
|
||||
cooldown.until(),
|
||||
RuntimeCooldownScope::Account,
|
||||
);
|
||||
}
|
||||
if let Ok(Some(cooldown)) = account_read
|
||||
&& cooldown.until() > now
|
||||
{
|
||||
// 账号级限流冷却跨凭据轮换保留:上游限流不因本地凭据更换解除。
|
||||
insert_runtime_cooldown(
|
||||
&mut cooled,
|
||||
account.id(),
|
||||
cooldown.until(),
|
||||
RuntimeCooldownScope::Account,
|
||||
);
|
||||
}
|
||||
if let Ok(Some(cooldown)) = model_read {
|
||||
if cooldown.credential_revision() != account.revision() {
|
||||
if cooldown.credential_revision() < account.revision() {
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.clear_scoped(account.id(), &model_scope, account.revision())
|
||||
.await;
|
||||
}
|
||||
} else if cooldown.until() > now {
|
||||
insert_runtime_cooldown(
|
||||
&mut cooled,
|
||||
account.id(),
|
||||
cooldown.until(),
|
||||
RuntimeCooldownScope::Model,
|
||||
);
|
||||
}
|
||||
if let Ok(Some(cooldown)) = model_read
|
||||
&& cooldown.until() > now
|
||||
{
|
||||
insert_runtime_cooldown(
|
||||
&mut cooled,
|
||||
account.id(),
|
||||
cooldown.until(),
|
||||
RuntimeCooldownScope::Model,
|
||||
);
|
||||
}
|
||||
}
|
||||
cooled
|
||||
@@ -374,7 +364,6 @@ impl GrokAccountSessionSelector {
|
||||
session: &SelectedGrokSession,
|
||||
availability: GrokCredentialAvailability,
|
||||
reason: &'static str,
|
||||
cooldown_until: Option<DateTime<Utc>>,
|
||||
observed_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
let update = |expected_revision| UpdateGrokCredentialState {
|
||||
@@ -382,7 +371,6 @@ impl GrokAccountSessionSelector {
|
||||
expected_revision,
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
};
|
||||
match self
|
||||
@@ -467,28 +455,33 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
GrokCredentialFailure::QuotaExhausted => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::FreeQuotaExhausted => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_free_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::PaymentRequired { .. } => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_payment_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::ModelQuotaExhausted { retry_after, .. } => {
|
||||
// bare 402 没有结构化 quota code,不能证明账号额度耗尽:
|
||||
// 只写短期账号级 runtime cooldown(TransientBackoff),
|
||||
// 不持久化 QuotaExhausted(结构化 QuotaExhausted/FreeQuotaExhausted 走上面)。
|
||||
GrokCredentialFailure::PaymentRequired { retry_after } => {
|
||||
let retry_after = retry_after
|
||||
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
return;
|
||||
}
|
||||
// xAI 免费模型额度按该模型滚动窗口恢复:限流写 model-scoped
|
||||
// runtime cooldown(Redis 跨重启保留),不进入持久化账号状态,
|
||||
// 不阻止该账号服务其他模型。
|
||||
GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model,
|
||||
retry_after,
|
||||
} => {
|
||||
let retry_after =
|
||||
bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN);
|
||||
// xAI 免费模型额度按账号滚动窗口恢复;持久化使页面和各调度节点收敛。
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"usage_limit_exhausted",
|
||||
persistent_cooldown_until(observed_at, retry_after),
|
||||
)
|
||||
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::ModelAccessDenied {
|
||||
upstream_model,
|
||||
@@ -508,31 +501,17 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"upstream_rate_limited",
|
||||
persistent_cooldown_until(observed_at, retry_after),
|
||||
)
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::StreamInterrupted => {
|
||||
self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN)
|
||||
.await;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"upstream_stream_interrupted",
|
||||
persistent_cooldown_until(observed_at, STREAM_INTERRUPTION_COOLDOWN),
|
||||
)
|
||||
return;
|
||||
}
|
||||
};
|
||||
let (availability, reason, cooldown_until) = persistent;
|
||||
self.persist_account_failure(
|
||||
session,
|
||||
availability,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
let (availability, reason) = persistent;
|
||||
self.persist_account_failure(session, availability, reason, observed_at)
|
||||
.await;
|
||||
})
|
||||
}
|
||||
|
||||
@@ -552,9 +531,7 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
|| !account.enabled()
|
||||
|| !matches!(
|
||||
account.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::Expired
|
||||
AccountAvailability::Unknown | AccountAvailability::Expired
|
||||
)
|
||||
{
|
||||
return;
|
||||
@@ -566,7 +543,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
expected_revision: account.revision(),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -600,15 +576,6 @@ fn bounded_cooldown(
|
||||
.min(maximum)
|
||||
}
|
||||
|
||||
fn persistent_cooldown_until(
|
||||
observed_at: DateTime<Utc>,
|
||||
duration: Duration,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
chrono::Duration::from_std(duration)
|
||||
.ok()
|
||||
.map(|duration| observed_at + duration)
|
||||
}
|
||||
|
||||
fn insert_runtime_cooldown(
|
||||
cooldowns: &mut std::collections::BTreeMap<ProviderAccountId, RuntimeCooldown>,
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -64,11 +64,13 @@ impl fmt::Debug for GrokAccountProfile {
|
||||
}
|
||||
|
||||
/// 与 `provider_accounts.availability` 一一对应的 xAI 状态。
|
||||
///
|
||||
/// 限流(429/滚动窗口)不进入持久化状态:由 `ProviderCooldownPort`(Redis 跨重启
|
||||
/// 保留)驱动调度排除,账号状态保持 `Ready`。
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum GrokCredentialAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
@@ -81,7 +83,6 @@ impl From<GrokCredentialAvailability> for AccountAvailability {
|
||||
match value {
|
||||
GrokCredentialAvailability::Unknown => Self::Unknown,
|
||||
GrokCredentialAvailability::Ready => Self::Ready,
|
||||
GrokCredentialAvailability::Cooldown => Self::Cooldown,
|
||||
GrokCredentialAvailability::QuotaExhausted => Self::QuotaExhausted,
|
||||
GrokCredentialAvailability::Expired => Self::Expired,
|
||||
GrokCredentialAvailability::Banned => Self::Banned,
|
||||
@@ -100,7 +101,6 @@ pub struct CreateGrokCredential {
|
||||
pub enabled: bool,
|
||||
pub initial_availability: GrokCredentialAvailability,
|
||||
pub initial_availability_reason: Option<String>,
|
||||
pub initial_cooldown_until: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for CreateGrokCredential {
|
||||
@@ -243,7 +243,6 @@ pub struct UpdateGrokCredentialState {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: GrokCredentialAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ pub async fn initialize(
|
||||
refresher,
|
||||
Arc::clone(&catalog),
|
||||
Arc::clone(&leases),
|
||||
cooldowns,
|
||||
cooldowns.clone(),
|
||||
ports.credential_state(),
|
||||
Arc::clone(&runtime_policy),
|
||||
));
|
||||
@@ -200,6 +200,7 @@ pub async fn initialize(
|
||||
quota: Arc::clone("a),
|
||||
catalog: Arc::clone(&catalog),
|
||||
runtime_policy,
|
||||
cooldowns: Arc::clone(&cooldowns),
|
||||
},
|
||||
cli_release_status,
|
||||
));
|
||||
|
||||
@@ -472,9 +472,8 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability: account.availability(),
|
||||
availability_reason: None,
|
||||
cooldown_until: account.cooldown_until().map(DateTime::<Utc>::from),
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -504,6 +503,7 @@ impl ProviderLeasePort for TestLeases {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -673,6 +673,13 @@ impl ProviderCooldownPort for TestCooldown {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
struct TestRuntimePolicy;
|
||||
|
||||
@@ -201,15 +201,13 @@ async fn set_account_state(
|
||||
store: &MemoryProviderAccountStore,
|
||||
id: &ProviderAccountId,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) {
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability,
|
||||
reason: Some("test isolation".to_owned()),
|
||||
cooldown_until,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -488,11 +486,10 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
|
||||
let id = account_id("quota-exhausted");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id,
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -686,18 +683,14 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_quota_refresh_releases_existing_quota_and_cooldown_isolation() {
|
||||
for (suffix, availability, cooldown_until) in [
|
||||
("recover-quota", AccountAvailability::QuotaExhausted, None),
|
||||
(
|
||||
"recover-cooldown",
|
||||
AccountAvailability::Cooldown,
|
||||
SystemTime::now().checked_add(Duration::from_secs(300)),
|
||||
),
|
||||
async fn recovered_quota_refresh_releases_existing_quota_isolation() {
|
||||
for (suffix, availability) in [
|
||||
("recover-quota", AccountAvailability::QuotaExhausted),
|
||||
("recover-ready", AccountAvailability::Ready),
|
||||
] {
|
||||
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
|
||||
let id = account_id(suffix);
|
||||
set_account_state(&store, &id, availability, cooldown_until).await;
|
||||
set_account_state(&store, &id, availability).await;
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":25}}"#),
|
||||
@@ -708,7 +701,6 @@ async fn recovered_quota_refresh_releases_existing_quota_and_cooldown_isolation(
|
||||
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.cooldown_until(), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -717,7 +709,7 @@ async fn authoritative_exhaustion_preserves_quota_exhausted_state() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await;
|
||||
let id = account_id("still-exhausted");
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted, None).await;
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
|
||||
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
@@ -742,7 +734,7 @@ async fn recovered_quota_does_not_clear_terminal_account_states() {
|
||||
] {
|
||||
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
|
||||
let id = account_id(suffix);
|
||||
set_account_state(&store, &id, availability, None).await;
|
||||
set_account_state(&store, &id, availability).await;
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":10}}"#),
|
||||
@@ -759,20 +751,16 @@ async fn recovered_quota_does_not_clear_terminal_account_states() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_refresh_does_not_overwrite_a_newer_cooldown() {
|
||||
async fn quota_refresh_preserves_ready_availability_across_concurrent_state_write() {
|
||||
let (store, repository) = repository_with_accounts(&[("new-cooldown", "new-cooldown")]).await;
|
||||
let id = account_id("new-cooldown");
|
||||
let new_cooldown = SystemTime::now()
|
||||
.checked_add(Duration::from_secs(600))
|
||||
.expect("cooldown time");
|
||||
let transport = Arc::new(MutatingBillingTransport {
|
||||
store: Arc::clone(&store),
|
||||
mutation: Mutex::new(Some(BillingMutation::State(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("new rate limit".to_owned()),
|
||||
cooldown_until: Some(new_cooldown),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
}))),
|
||||
body: br#"{"config":{"creditUsagePercent":10}}"#.to_vec(),
|
||||
@@ -781,11 +769,10 @@ async fn quota_refresh_does_not_overwrite_a_newer_cooldown() {
|
||||
crate::support::grok_quota_service(repository, transport)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh around concurrent cooldown");
|
||||
.expect("refresh around concurrent state write");
|
||||
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(account.cooldown_until(), Some(new_cooldown));
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -878,7 +865,7 @@ async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustio
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("free-quota", "subject-free-quota")]).await;
|
||||
let id = account_id("free-quota");
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted, None).await;
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#,
|
||||
);
|
||||
@@ -935,6 +922,7 @@ async fn quota_read_rejects_corrupt_provider_document() {
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
quota: Some(OpaqueProviderData::new(document)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed corrupt quota");
|
||||
|
||||
@@ -219,44 +219,18 @@ async fn state_update_uses_credential_revision_fence() {
|
||||
let (store, repository) = repository();
|
||||
let input = create_input("state", "subject-state");
|
||||
seed_input(&store, &input).await.expect("create");
|
||||
let cooldown_until = Utc::now() + chrono::Duration::minutes(1);
|
||||
repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Cooldown,
|
||||
availability_reason: Some("rate_limited".to_owned()),
|
||||
cooldown_until: Some(cooldown_until),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: Some("upstream_rate_limited".to_owned()),
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.expect("state update");
|
||||
let account = store.account(&input.account_id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(account.cooldown_until().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_requires_matching_deadline() {
|
||||
let (store, repository) = repository();
|
||||
let input = create_input("bad-cooldown", "subject-cooldown");
|
||||
seed_input(&store, &input).await.expect("create");
|
||||
let result = repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id,
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Cooldown,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
result,
|
||||
Err(GrokCredentialRepositoryError::InvalidInput(
|
||||
"cooldown_until"
|
||||
))
|
||||
);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -420,7 +420,7 @@ struct TestKey {
|
||||
impl TestKey {
|
||||
fn new(seed: u8, kid: &'static str) -> Self {
|
||||
let signing_key = SigningKey::from_slice(&[seed; 32]).expect("valid deterministic scalar");
|
||||
let point = signing_key.verifying_key().to_encoded_point(false);
|
||||
let point = signing_key.verifying_key().to_sec1_point(false);
|
||||
let x = URL_SAFE_NO_PAD.encode(point.x().expect("P-256 x coordinate"));
|
||||
let y = URL_SAFE_NO_PAD.encode(point.y().expect("P-256 y coordinate"));
|
||||
Self {
|
||||
|
||||
@@ -10,8 +10,8 @@ use gateway_core::engine::credential::{
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCredentialState, ProviderCredentialStatePort, ProviderLeaseAcquisition,
|
||||
ProviderLeasePort, ProviderLeaseRequest, ProviderStoreError,
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCredentialState, ProviderCredentialStatePort,
|
||||
ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, ProviderStoreError,
|
||||
};
|
||||
use provider_xai::{
|
||||
GrokCredentialCatalogCache, GrokCredentialRecovery, GrokCredentialRecoveryOutcome,
|
||||
@@ -143,6 +143,48 @@ async fn fixture(
|
||||
fixture_many([input], responses, lease_available).await
|
||||
}
|
||||
|
||||
/// 带外部注入 cooldown 端口的 fixture(AUD-05 组合测试用)。
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn fixture_with_cooldowns(
|
||||
input: provider_xai::CreateGrokCredential,
|
||||
responses: impl IntoIterator<Item = Result<GrokRefreshTokens, GrokRefreshFailure>>,
|
||||
lease_available: bool,
|
||||
cooldowns: Arc<MemoryCooldownPort>,
|
||||
) -> (
|
||||
Arc<MemoryProviderAccountStore>,
|
||||
GrokCredentialRepository,
|
||||
Arc<QueueRefresher>,
|
||||
GrokCredentialRefreshService,
|
||||
Arc<MemoryCooldownPort>,
|
||||
) {
|
||||
let store = MemoryProviderAccountStore::shared();
|
||||
let account_store: Arc<dyn ProviderAccountStore> = store.clone();
|
||||
let repository = GrokCredentialRepository::new(account_store);
|
||||
seed_input(&store, &input).await.expect("create account");
|
||||
let refresher = QueueRefresher::new(responses);
|
||||
let refresher_port: Arc<dyn GrokCredentialRefresher> = refresher.clone();
|
||||
let cache: Arc<dyn GrokCredentialCatalogCache> = MemoryGrokCatalogCache::shared();
|
||||
let catalog = Arc::new(crate::support::grok_catalog_service(
|
||||
repository.clone(),
|
||||
Arc::new(StaticCatalogTransport),
|
||||
cache,
|
||||
));
|
||||
let leases = Arc::new(TestRefreshLeases {
|
||||
available: lease_available,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let service = GrokCredentialRefreshService::new(
|
||||
repository.clone(),
|
||||
refresher_port,
|
||||
catalog,
|
||||
leases,
|
||||
cooldowns.clone(),
|
||||
Arc::new(CountingCredentialState::default()),
|
||||
runtime_policy(),
|
||||
);
|
||||
(store, repository, refresher, service, cooldowns)
|
||||
}
|
||||
|
||||
/// 真实累加连续失败计数的测试 double,用于断言退避的指数增长与成功清零。
|
||||
#[derive(Default)]
|
||||
struct CountingCredentialState {
|
||||
@@ -732,3 +774,54 @@ async fn malformed_account_refresh_does_not_stop_later_accounts() {
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_cooldown_survives_credential_rotation_and_blocks_refresh() {
|
||||
// cooldown 写于 revision N,轮换到 N+1 后 refresh worker 不得清除
|
||||
// 或失效它;cooldown 活跃期间账号被跳过刷新。
|
||||
let input = create_input(
|
||||
"cooldown-survives-rotation",
|
||||
"subject-cooldown-survives-rotation",
|
||||
);
|
||||
let id = input.account_id.clone();
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::default());
|
||||
let (store, _, _, service, cooldowns) = fixture_with_cooldowns(
|
||||
input,
|
||||
[Ok(success_tokens(Some("rotated-refresh")))],
|
||||
true,
|
||||
cooldowns,
|
||||
)
|
||||
.await;
|
||||
// 先写 account cooldown(revision 1,未来到期)。
|
||||
let until = SystemTime::now() + Duration::from_secs(3600);
|
||||
cooldowns
|
||||
.put_if_later(ProviderCooldown::new(
|
||||
id.clone(),
|
||||
CredentialRevision::new(1).expect("revision"),
|
||||
until,
|
||||
))
|
||||
.await
|
||||
.expect("write cooldown");
|
||||
|
||||
// 轮换到 revision 2。
|
||||
let outcome = service
|
||||
.recover_unauthorized(&id, CredentialRevision::new(1).expect("revision"))
|
||||
.await;
|
||||
assert_eq!(outcome, GrokCredentialRecoveryOutcome::Recovered);
|
||||
assert_eq!(store.account(&id).expect("account").revision().get(), 2);
|
||||
|
||||
// cooldown 仍有效(轮换不清除)。
|
||||
assert!(
|
||||
cooldowns
|
||||
.cooldown(&id)
|
||||
.is_some_and(|cooldown| cooldown.until() > SystemTime::now()),
|
||||
"rotation must not clear account runtime cooldown"
|
||||
);
|
||||
// cooldown 活跃期间 refresh worker 跳过该账号(不清除、不刷新)。
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
assert!(
|
||||
outcomes.is_empty(),
|
||||
"cooldown-active account must be skipped by refresh worker"
|
||||
);
|
||||
assert_eq!(store.account(&id).expect("account").revision().get(), 2);
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ use gateway_core::engine::credential::{
|
||||
QuotaWriteOutcome, RotationStrategy,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderSchedulingState, ProviderStoreError,
|
||||
ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
ProviderLeaseRequest, ProviderSchedulingState, ProviderStoreError,
|
||||
};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use provider_xai::{
|
||||
@@ -110,7 +110,6 @@ impl SelectorFixture {
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -132,6 +131,7 @@ impl SelectorFixture {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -245,6 +245,7 @@ impl SelectorFixture {
|
||||
document.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -369,21 +370,17 @@ async fn account_scoped_cooldown_survives_credential_rotation() {
|
||||
CredentialCasOutcome::Updated(revision) if revision.get() == 2
|
||||
));
|
||||
|
||||
// 账号级限流冷却跨凭据轮换保留:轮换后仍被冷却排除。
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await,
|
||||
Err(GrokSessionSelectorError::NoEligibleSession)
|
||||
Err(GrokSessionSelectorError::AccountCoolingDown { .. })
|
||||
));
|
||||
let account = fixture.store.account(&id).expect("rotated account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -405,12 +402,7 @@ async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() {
|
||||
)
|
||||
.await;
|
||||
let account = fixture.store.account(&selected).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -450,11 +442,10 @@ async fn successful_request_clears_the_persisted_cooldown_state() {
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(account.cooldown_until().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn payment_required_feedback_persists_account_quota_exhaustion() {
|
||||
async fn payment_required_feedback_writes_short_account_cooldown_without_persisting_exhaustion() {
|
||||
let fixture = SelectorFixture::new(&["payment-required", "available"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
@@ -472,15 +463,22 @@ async fn payment_required_feedback_persists_account_quota_exhaustion() {
|
||||
)
|
||||
.await;
|
||||
|
||||
// bare 402 无结构化 quota code:只写短期账号 runtime cooldown,
|
||||
// 不持久化 QuotaExhausted(避免长期错误状态)。
|
||||
assert_eq!(
|
||||
fixture
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("selected account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.cooldown(&selected)
|
||||
.is_some_and(|cooldown| cooldown.until() > SystemTime::now())
|
||||
);
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_none());
|
||||
let next = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
@@ -490,7 +488,7 @@ async fn payment_required_feedback_persists_account_quota_exhaustion() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_model_quota_feedback_persists_resettable_account_exhaustion() {
|
||||
async fn model_quota_feedback_writes_model_scoped_cooldown_without_blocking_other_models() {
|
||||
let fixture = SelectorFixture::new(&["model-quota", "available"]).await;
|
||||
let failed_model = UpstreamModelId::new("grok-4.5").expect("failed model");
|
||||
let session = fixture
|
||||
@@ -511,41 +509,34 @@ async fn free_model_quota_feedback_persists_resettable_account_exhaustion() {
|
||||
.await;
|
||||
|
||||
let account = fixture.store.account(&selected).expect("selected account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > minimum_until)
|
||||
);
|
||||
let scope = ProviderCooldownScope::upstream_model(failed_model);
|
||||
// model-scoped cooldown:目标模型被排除,账号级无 cooldown。
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.scoped_cooldown(&selected, &scope)
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.cooldown(&selected)
|
||||
.is_some_and(|cooldown| cooldown.until() > minimum_until)
|
||||
);
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_none());
|
||||
// 失败模型不可选(ModelCoolingDown 由 model-scoped cooldown 派生)。
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.selector
|
||||
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
|
||||
.select(fixture.request_for_model("grok-4.5", Some(selected.clone())))
|
||||
.await,
|
||||
Err(GrokSessionSelectorError::AccountCoolingDown {
|
||||
Err(GrokSessionSelectorError::ModelCoolingDown {
|
||||
retry_after: Some(_)
|
||||
})
|
||||
));
|
||||
let next = fixture
|
||||
// 另一模型不受 model cooldown 影响:同一账号在 grok-4.6 下仍可选。
|
||||
let other_model = fixture
|
||||
.selector
|
||||
.select(fixture.request_for_model("grok-4.6", None))
|
||||
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
|
||||
.await
|
||||
.expect("another account remains schedulable");
|
||||
assert_ne!(next.account_id(), &selected);
|
||||
.expect("model-scoped cooldown must not block other models");
|
||||
assert_eq!(other_model.account_id(), &selected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -604,12 +595,7 @@ async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() {
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -991,3 +977,111 @@ async fn quota_reset_strategy_uses_provider_reported_earliest_reset() {
|
||||
|
||||
assert_eq!(session.account_id(), &account_id("zzz-earlier-reset"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleting_account_clears_all_account_and_model_scoped_cooldowns() {
|
||||
// 删除账号后其 account/model scoped cooldown key 全部清除。
|
||||
let fixture = SelectorFixture::new(&["clear-cooldowns", "available"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("session");
|
||||
let selected = session.account_id().clone();
|
||||
let model = UpstreamModelId::new("grok-4.5").expect("model");
|
||||
// 写 account cooldown + model scoped cooldown。
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(
|
||||
&session,
|
||||
GrokCredentialFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(60)),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(
|
||||
&session,
|
||||
GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model: model.clone(),
|
||||
retry_after: Some(Duration::from_secs(3600)),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_some());
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.scoped_cooldown(
|
||||
&selected,
|
||||
&ProviderCooldownScope::upstream_model(model.clone())
|
||||
)
|
||||
.is_some()
|
||||
);
|
||||
|
||||
// 删除账号(clear_all 清全部)。
|
||||
fixture
|
||||
.cooldowns
|
||||
.clear_all(&selected)
|
||||
.await
|
||||
.expect("clear all cooldowns");
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_none());
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.scoped_cooldown(
|
||||
&selected,
|
||||
&ProviderCooldownScope::upstream_model(model.clone())
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bare_402_cooldown_expires_and_account_recovers_without_persisted_exhaustion() {
|
||||
// bare 402 只写短期 account cooldown,不持久化
|
||||
// QuotaExhausted;退避到期后账号自动恢复可选。
|
||||
let fixture = SelectorFixture::new(&["payment-recover", "available"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("session");
|
||||
let selected = session.account_id().clone();
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(
|
||||
&session,
|
||||
GrokCredentialFailure::PaymentRequired {
|
||||
retry_after: Some(Duration::from_secs(1)),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
// 退避活跃期间该账号被排除,另一账号可选。
|
||||
assert_eq!(
|
||||
fixture
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
"bare 402 must not persist QuotaExhausted"
|
||||
);
|
||||
let next = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("another account available during cooldown");
|
||||
assert_ne!(next.account_id(), &selected);
|
||||
|
||||
// 等短 cooldown 到期。
|
||||
tokio::time::sleep(Duration::from_millis(1_100)).await;
|
||||
let recovered = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("account recovers after cooldown expiry");
|
||||
assert_eq!(recovered.account_id(), &selected);
|
||||
}
|
||||
|
||||
@@ -328,7 +328,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
revision: next,
|
||||
access_token_expires_at,
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token,
|
||||
next_refresh_at,
|
||||
@@ -381,7 +380,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: change.availability,
|
||||
cooldown_until: change.cooldown_until,
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -402,7 +400,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -427,7 +424,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: stored.account.availability(),
|
||||
cooldown_until: stored.account.cooldown_until(),
|
||||
enabled,
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -449,7 +445,6 @@ struct AccountReplacement {
|
||||
revision: CredentialRevision,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
enabled: bool,
|
||||
has_refresh_token: bool,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
@@ -473,11 +468,7 @@ fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement)
|
||||
previous.upstream_account_id().map(str::to_owned),
|
||||
replacement.plan_type,
|
||||
)
|
||||
.with_runtime_state(
|
||||
replacement.enabled,
|
||||
replacement.availability,
|
||||
replacement.cooldown_until,
|
||||
)
|
||||
.with_runtime_state(replacement.enabled, replacement.availability)
|
||||
.with_refresh_schedule(replacement.has_refresh_token, replacement.next_refresh_at)
|
||||
}
|
||||
|
||||
@@ -549,7 +540,6 @@ pub fn create_input(suffix: &str, subject: &str) -> CreateGrokCredential {
|
||||
enabled: true,
|
||||
initial_availability: GrokCredentialAvailability::Unknown,
|
||||
initial_availability_reason: None,
|
||||
initial_cooldown_until: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -734,6 +724,20 @@ impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
Ok(should_remove)
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut cooldowns = lock(&self.cooldowns);
|
||||
let mut scoped = lock(&self.scoped_cooldowns);
|
||||
let account_removed = cooldowns.remove(account_id).is_some();
|
||||
let scoped_removed = scoped.keys().any(|(id, _)| id == account_id);
|
||||
scoped.retain(|(id, _), _| id != account_id);
|
||||
Ok(account_removed || scoped_removed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderRuntimePolicyPort for StaticRuntimePolicy {
|
||||
|
||||
@@ -1,4 +1 @@
|
||||
a4681a5c040e80c5234e879293fc34972080e5251dfbc7c40e1374f8e4170a4c 0001_initial.sql
|
||||
7dc3b626e7a958281f82c0e646b8284f8b373e0c7d3dbdaf750811ce0c4a7f83 0002_snapshot_provider_account_identity.sql
|
||||
ada40b11728ba67cfe44d8492bc77a4754b36640e575b25daa5dfe90e92e43aa 0003_s3_backup.sql
|
||||
52087c48b289eb5d8d9b00e338bb4d9861e81290e0be6bd0e311862ba2d4b59b 0004_drop_provider_accounts_id_kind_uq.sql
|
||||
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
-- 管理员身份与密码摘要。
|
||||
create table admin_users (
|
||||
id text primary key,
|
||||
password_hash text not null,
|
||||
@@ -6,6 +7,7 @@ create table admin_users (
|
||||
constraint admin_users_time_ck check (created_at <= updated_at)
|
||||
);
|
||||
|
||||
-- 管理操作审计事件。
|
||||
create table admin_audit_events (
|
||||
id text primary key,
|
||||
actor_kind text not null,
|
||||
@@ -44,6 +46,7 @@ create index admin_audit_events_entity_idx
|
||||
create index admin_audit_events_actor_ref_idx
|
||||
on admin_audit_events (actor_ref, created_at desc, id desc);
|
||||
|
||||
-- 客户端 API 密钥、鉴权范围与限额。
|
||||
create table client_api_keys (
|
||||
id text primary key,
|
||||
name text not null,
|
||||
@@ -74,6 +77,7 @@ create index client_api_keys_created_idx
|
||||
create index client_api_keys_last_used_idx
|
||||
on client_api_keys (last_used_at desc, id desc);
|
||||
|
||||
-- 全局运行参数与配置版本。
|
||||
create table runtime_settings (
|
||||
id bigint primary key,
|
||||
config_revision bigint not null default 1,
|
||||
@@ -110,6 +114,7 @@ create table runtime_settings (
|
||||
)
|
||||
);
|
||||
|
||||
-- 上游 Provider 账号、凭据与可用性状态。
|
||||
create table provider_accounts (
|
||||
id text primary key,
|
||||
provider_kind text not null,
|
||||
@@ -126,9 +131,9 @@ create table provider_accounts (
|
||||
next_refresh_at timestamptz,
|
||||
enabled boolean not null default true,
|
||||
availability text not null default 'unknown',
|
||||
availability_reason text,
|
||||
cooldown_until timestamptz,
|
||||
provider_quota_json jsonb,
|
||||
quota_limit_reached boolean not null default false,
|
||||
last_error_message text,
|
||||
availability_observed_at timestamptz not null,
|
||||
quota_observed_at timestamptz,
|
||||
created_at timestamptz not null,
|
||||
@@ -158,23 +163,12 @@ create table provider_accounts (
|
||||
availability in (
|
||||
'unknown',
|
||||
'ready',
|
||||
'cooldown',
|
||||
'quota_exhausted',
|
||||
'expired',
|
||||
'banned',
|
||||
'invalid'
|
||||
)
|
||||
),
|
||||
constraint provider_accounts_cooldown_ck check (
|
||||
(
|
||||
availability != 'cooldown'
|
||||
or cooldown_until is not null
|
||||
)
|
||||
and (
|
||||
cooldown_until is null
|
||||
or availability in ('cooldown', 'quota_exhausted')
|
||||
)
|
||||
),
|
||||
constraint provider_accounts_time_ck check (
|
||||
created_at <= updated_at
|
||||
and availability_observed_at <= updated_at
|
||||
@@ -182,8 +176,6 @@ create table provider_accounts (
|
||||
)
|
||||
);
|
||||
|
||||
create unique index provider_accounts_id_kind_uq
|
||||
on provider_accounts (id, provider_kind);
|
||||
create unique index provider_accounts_upstream_identity_uq
|
||||
on provider_accounts (
|
||||
provider_kind,
|
||||
@@ -193,7 +185,7 @@ create unique index provider_accounts_upstream_identity_uq
|
||||
create index provider_accounts_runtime_idx
|
||||
on provider_accounts (provider_kind, enabled, id);
|
||||
create index provider_accounts_availability_idx
|
||||
on provider_accounts (availability, cooldown_until, id);
|
||||
on provider_accounts (availability, id);
|
||||
create index provider_accounts_access_expiry_idx
|
||||
on provider_accounts (access_token_expires_at, id);
|
||||
create index provider_accounts_refresh_due_idx
|
||||
@@ -203,6 +195,7 @@ create index provider_accounts_email_idx
|
||||
on provider_accounts (provider_kind, lower(email))
|
||||
where email is not null;
|
||||
|
||||
-- 模型请求、上游尝试、用量、计费与结果。
|
||||
create table model_requests (
|
||||
id text primary key,
|
||||
client_api_key_id text,
|
||||
@@ -218,6 +211,9 @@ create table model_requests (
|
||||
service_tier text,
|
||||
provider_account_id text,
|
||||
provider_account_ref text,
|
||||
provider_account_name_snapshot text,
|
||||
provider_account_email_snapshot text,
|
||||
provider_account_authentication_kind_snapshot text,
|
||||
upstream_transport text,
|
||||
http_version text,
|
||||
websocket_pool text,
|
||||
@@ -228,8 +224,6 @@ create table model_requests (
|
||||
outcome text not null default 'running',
|
||||
client_status_code integer,
|
||||
upstream_status_code integer,
|
||||
-- Provider response IDs are opaque JSON strings. Bytes preserve embedded NUL
|
||||
-- and avoid imposing PostgreSQL text-index limits on protocol handles.
|
||||
client_response_id bytea,
|
||||
upstream_request_id text,
|
||||
upstream_response_id bytea,
|
||||
@@ -431,6 +425,8 @@ create index model_requests_running_deadline_idx
|
||||
create index model_requests_retention_idx
|
||||
on model_requests (completed_at)
|
||||
where outcome <> 'running';
|
||||
|
||||
-- 运行期警告与错误事件。
|
||||
create table ops_events (
|
||||
id text primary key,
|
||||
model_request_id text,
|
||||
@@ -441,6 +437,9 @@ create table ops_events (
|
||||
provider_kind text,
|
||||
provider_account_id text,
|
||||
provider_account_ref text,
|
||||
provider_account_name_snapshot text,
|
||||
provider_account_email_snapshot text,
|
||||
provider_account_authentication_kind_snapshot text,
|
||||
upstream_model_id text,
|
||||
failure_kind text not null,
|
||||
status_code integer,
|
||||
@@ -506,3 +505,183 @@ create index ops_events_retention_idx
|
||||
|
||||
insert into runtime_settings (id, updated_at)
|
||||
values (1, now());
|
||||
|
||||
-- S3 兼容对象存储、备份计划与保留策略。
|
||||
create table backup_settings (
|
||||
id bigint primary key,
|
||||
storage_revision bigint not null default 1,
|
||||
endpoint text,
|
||||
region text,
|
||||
bucket text,
|
||||
access_key_id text,
|
||||
secret_access_key text,
|
||||
prefix text,
|
||||
force_path_style boolean not null default false,
|
||||
schedule_enabled boolean not null default false,
|
||||
cron_expression text,
|
||||
schedule_timezone text default 'Asia/Shanghai',
|
||||
retention_days bigint not null default 0,
|
||||
retention_count bigint not null default 0,
|
||||
next_run_at timestamptz,
|
||||
last_verified_at timestamptz,
|
||||
updated_at timestamptz not null,
|
||||
constraint backup_settings_singleton_ck check (id = 1),
|
||||
constraint backup_settings_revision_ck check (storage_revision > 0),
|
||||
constraint backup_settings_endpoint_ck check (
|
||||
endpoint is null
|
||||
or (
|
||||
endpoint ~ '^https?://'
|
||||
and endpoint !~ '[[:space:][:cntrl:]]'
|
||||
and endpoint !~ '://[^/]*@'
|
||||
)
|
||||
),
|
||||
constraint backup_settings_lengths_ck check (
|
||||
(bucket is null or (octet_length(bucket) between 1 and 255 and bucket !~ '[[:cntrl:]]'))
|
||||
and (region is null or (octet_length(region) between 1 and 64 and region !~ '[[:cntrl:]]'))
|
||||
and (prefix is null or (octet_length(prefix) <= 1024 and prefix !~ '[[:cntrl:]]'))
|
||||
and (
|
||||
access_key_id is null
|
||||
or (octet_length(access_key_id) between 1 and 255 and access_key_id !~ '[[:cntrl:]]')
|
||||
)
|
||||
and (
|
||||
secret_access_key is null
|
||||
or (
|
||||
octet_length(secret_access_key) between 1 and 512
|
||||
and secret_access_key !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint backup_settings_schedule_ck check (
|
||||
(cron_expression is null or cron_expression ~ '^[^[:space:]]+([[:space:]]+[^[:space:]]+){4}$')
|
||||
and (
|
||||
schedule_timezone is null
|
||||
or (octet_length(schedule_timezone) between 1 and 64 and schedule_timezone !~ '[[:cntrl:]]')
|
||||
)
|
||||
and retention_days >= 0
|
||||
and retention_count >= 0
|
||||
and (
|
||||
not schedule_enabled
|
||||
or (
|
||||
endpoint is not null
|
||||
and bucket is not null
|
||||
and region is not null
|
||||
and access_key_id is not null
|
||||
and secret_access_key is not null
|
||||
and prefix is not null
|
||||
and cron_expression is not null
|
||||
and schedule_timezone is not null
|
||||
and last_verified_at is not null
|
||||
)
|
||||
)
|
||||
),
|
||||
-- 计划启停与游标一致性:启用必有 next_run_at,关闭必清空。
|
||||
constraint backup_settings_cursor_ck check (
|
||||
(not schedule_enabled and next_run_at is null)
|
||||
or (schedule_enabled and next_run_at is not null)
|
||||
)
|
||||
);
|
||||
|
||||
insert into backup_settings (
|
||||
id,
|
||||
storage_revision,
|
||||
schedule_enabled,
|
||||
retention_days,
|
||||
retention_count,
|
||||
force_path_style,
|
||||
schedule_timezone,
|
||||
updated_at
|
||||
)
|
||||
values (1, 1, false, 0, 0, false, 'Asia/Shanghai', now());
|
||||
|
||||
-- 备份任务状态与归档信息。
|
||||
create table backup_records (
|
||||
id text primary key,
|
||||
trigger_kind text not null,
|
||||
status text not null,
|
||||
scheduled_at timestamptz,
|
||||
object_key text not null,
|
||||
size_bytes bigint,
|
||||
sha256 text,
|
||||
expires_at timestamptz,
|
||||
attempt_count integer not null default 0,
|
||||
error_code text,
|
||||
error_message text,
|
||||
started_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz not null,
|
||||
updated_at timestamptz not null,
|
||||
constraint backup_records_id_ck check (id ~ '^backup_[0-9a-f]{32}$'),
|
||||
constraint backup_records_trigger_kind_ck check (
|
||||
trigger_kind in ('manual', 'scheduled')
|
||||
and (
|
||||
(trigger_kind = 'manual' and scheduled_at is null)
|
||||
or (trigger_kind = 'scheduled' and scheduled_at is not null)
|
||||
)
|
||||
),
|
||||
constraint backup_records_status_ck check (
|
||||
status in ('queued', 'dumping', 'uploading', 'completed', 'failed', 'deleting')
|
||||
),
|
||||
constraint backup_records_attempt_ck check (attempt_count >= 0),
|
||||
constraint backup_records_lifecycle_ck check (
|
||||
created_at <= updated_at
|
||||
and (started_at is null or started_at >= created_at)
|
||||
and (completed_at is null or completed_at >= created_at)
|
||||
and (
|
||||
(
|
||||
status = 'queued'
|
||||
and started_at is null
|
||||
and completed_at is null
|
||||
)
|
||||
or (
|
||||
status in ('dumping', 'uploading')
|
||||
and started_at is not null
|
||||
and completed_at is null
|
||||
)
|
||||
or (
|
||||
status in ('completed', 'failed', 'deleting')
|
||||
and started_at is not null
|
||||
and completed_at is not null
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint backup_records_artifact_ck check (
|
||||
(
|
||||
status not in ('uploading', 'completed')
|
||||
or (size_bytes is not null and sha256 is not null)
|
||||
)
|
||||
and (size_bytes is null or size_bytes >= 0)
|
||||
and (sha256 is null or sha256 ~ '^[0-9a-f]{64}$')
|
||||
),
|
||||
constraint backup_records_error_ck check (
|
||||
(error_code is null) = (error_message is null)
|
||||
and (status <> 'failed' or (error_code is not null and error_message is not null))
|
||||
and (
|
||||
status not in ('queued', 'dumping', 'uploading', 'completed')
|
||||
or error_code is null
|
||||
)
|
||||
),
|
||||
-- expires_at 为空,或不早于创建时间(创建时 now + 天数 恒晚于 created_at)。
|
||||
constraint backup_records_expiry_ck check (
|
||||
expires_at is null or expires_at >= created_at
|
||||
)
|
||||
);
|
||||
|
||||
create index backup_records_created_idx
|
||||
on backup_records (created_at desc, id desc);
|
||||
create index backup_records_status_idx
|
||||
on backup_records (status, created_at, id);
|
||||
create unique index backup_records_scheduled_uq
|
||||
on backup_records (scheduled_at)
|
||||
where trigger_kind = 'scheduled' and scheduled_at is not null;
|
||||
create unique index backup_records_active_uq
|
||||
on backup_records ((1))
|
||||
where status in ('queued', 'dumping', 'uploading');
|
||||
create unique index backup_records_object_key_uq
|
||||
on backup_records (object_key)
|
||||
where object_key is not null;
|
||||
create index backup_records_scheduled_completed_idx
|
||||
on backup_records (completed_at desc, id desc)
|
||||
where trigger_kind = 'scheduled' and status = 'completed';
|
||||
create index backup_records_retention_idx
|
||||
on backup_records (expires_at)
|
||||
where expires_at is not null and status in ('completed', 'failed');
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
-- Provider 账号观测快照:请求/事件发生时账号的名称、邮箱与认证类型。
|
||||
-- 账号删除后 provider_account_id 外键置空,历史显示仍依赖这些快照列;
|
||||
-- provider_account_ref 继续承担稳定筛选与关联事实。
|
||||
|
||||
alter table model_requests
|
||||
add column provider_account_name_snapshot text,
|
||||
add column provider_account_email_snapshot text,
|
||||
add column provider_account_authentication_kind_snapshot text;
|
||||
|
||||
alter table ops_events
|
||||
add column provider_account_name_snapshot text,
|
||||
add column provider_account_email_snapshot text,
|
||||
add column provider_account_authentication_kind_snapshot text;
|
||||
|
||||
-- 迁移时仍存在账号的历史行回填快照;已删除账号的历史行保持为空,
|
||||
-- 读取侧回退 provider_account_ref。迁移前已删除账号的资料无法恢复。
|
||||
update model_requests mr
|
||||
set provider_account_name_snapshot = pa.name,
|
||||
provider_account_email_snapshot = pa.email,
|
||||
provider_account_authentication_kind_snapshot = pa.authentication_kind
|
||||
from provider_accounts pa
|
||||
where pa.id = mr.provider_account_ref;
|
||||
|
||||
update ops_events oe
|
||||
set provider_account_name_snapshot = pa.name,
|
||||
provider_account_email_snapshot = pa.email,
|
||||
provider_account_authentication_kind_snapshot = pa.authentication_kind
|
||||
from provider_accounts pa
|
||||
where pa.id = oe.provider_account_ref;
|
||||
@@ -1,189 +0,0 @@
|
||||
-- S3 兼容对象存储(含 Cloudflare R2)与 PostgreSQL 备份。
|
||||
-- 按 docs/s3-backup-design-audit.md 最终状态实现。
|
||||
--
|
||||
-- backup_settings 是单例配置行(id = 1);backup_records 记录每次备份任务。
|
||||
-- 当前部署边界是单副本:任务执行由单个可取消 Daemon 承担,记录表不需要
|
||||
-- fencing token、heartbeat 或墓碑状态。skipped 只是日志/指标,删除成功后记录
|
||||
-- 硬删除,操作历史进入 admin_audit_events。
|
||||
-- 手动过期 TTL:每份备份独立的 expires_at,创建时确定,到期自动清理
|
||||
-- (审计 A-06 曾移除该字段,按参考实现 sub2api 的语义加回)。
|
||||
|
||||
create table backup_settings (
|
||||
id bigint primary key,
|
||||
storage_revision bigint not null default 1,
|
||||
endpoint text,
|
||||
region text,
|
||||
bucket text,
|
||||
access_key_id text,
|
||||
secret_access_key text,
|
||||
prefix text,
|
||||
force_path_style boolean not null default false,
|
||||
schedule_enabled boolean not null default false,
|
||||
cron_expression text,
|
||||
schedule_timezone text default 'Asia/Shanghai',
|
||||
retention_days bigint not null default 0,
|
||||
retention_count bigint not null default 0,
|
||||
next_run_at timestamptz,
|
||||
last_verified_at timestamptz,
|
||||
updated_at timestamptz not null,
|
||||
constraint backup_settings_singleton_ck check (id = 1),
|
||||
constraint backup_settings_revision_ck check (storage_revision > 0),
|
||||
constraint backup_settings_endpoint_ck check (
|
||||
endpoint is null
|
||||
or (
|
||||
endpoint ~ '^https?://'
|
||||
and endpoint !~ '[[:space:][:cntrl:]]'
|
||||
and endpoint !~ '://[^/]*@'
|
||||
)
|
||||
),
|
||||
constraint backup_settings_lengths_ck check (
|
||||
(bucket is null or (octet_length(bucket) between 1 and 255 and bucket !~ '[[:cntrl:]]'))
|
||||
and (region is null or (octet_length(region) between 1 and 64 and region !~ '[[:cntrl:]]'))
|
||||
and (prefix is null or (octet_length(prefix) <= 1024 and prefix !~ '[[:cntrl:]]'))
|
||||
and (
|
||||
access_key_id is null
|
||||
or (octet_length(access_key_id) between 1 and 255 and access_key_id !~ '[[:cntrl:]]')
|
||||
)
|
||||
and (
|
||||
secret_access_key is null
|
||||
or (
|
||||
octet_length(secret_access_key) between 1 and 512
|
||||
and secret_access_key !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint backup_settings_schedule_ck check (
|
||||
(cron_expression is null or cron_expression ~ '^[^[:space:]]+([[:space:]]+[^[:space:]]+){4}$')
|
||||
and (
|
||||
schedule_timezone is null
|
||||
or (octet_length(schedule_timezone) between 1 and 64 and schedule_timezone !~ '[[:cntrl:]]')
|
||||
)
|
||||
and retention_days >= 0
|
||||
and retention_count >= 0
|
||||
and (
|
||||
not schedule_enabled
|
||||
or (
|
||||
endpoint is not null
|
||||
and bucket is not null
|
||||
and region is not null
|
||||
and access_key_id is not null
|
||||
and secret_access_key is not null
|
||||
and prefix is not null
|
||||
and cron_expression is not null
|
||||
and schedule_timezone is not null
|
||||
and last_verified_at is not null
|
||||
)
|
||||
)
|
||||
),
|
||||
-- 计划启停与游标一致性:启用必有 next_run_at,关闭必清空。
|
||||
constraint backup_settings_cursor_ck check (
|
||||
(not schedule_enabled and next_run_at is null)
|
||||
or (schedule_enabled and next_run_at is not null)
|
||||
)
|
||||
);
|
||||
|
||||
insert into backup_settings (
|
||||
id,
|
||||
storage_revision,
|
||||
schedule_enabled,
|
||||
retention_days,
|
||||
retention_count,
|
||||
force_path_style,
|
||||
schedule_timezone,
|
||||
updated_at
|
||||
)
|
||||
values (1, 1, false, 0, 0, false, 'Asia/Shanghai', now());
|
||||
|
||||
create table backup_records (
|
||||
id text primary key,
|
||||
trigger_kind text not null,
|
||||
status text not null,
|
||||
scheduled_at timestamptz,
|
||||
object_key text not null,
|
||||
size_bytes bigint,
|
||||
sha256 text,
|
||||
expires_at timestamptz,
|
||||
attempt_count integer not null default 0,
|
||||
error_code text,
|
||||
error_message text,
|
||||
started_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz not null,
|
||||
updated_at timestamptz not null,
|
||||
constraint backup_records_id_ck check (id ~ '^backup_[0-9a-f]{32}$'),
|
||||
constraint backup_records_trigger_kind_ck check (
|
||||
trigger_kind in ('manual', 'scheduled')
|
||||
and (
|
||||
(trigger_kind = 'manual' and scheduled_at is null)
|
||||
or (trigger_kind = 'scheduled' and scheduled_at is not null)
|
||||
)
|
||||
),
|
||||
constraint backup_records_status_ck check (
|
||||
status in ('queued', 'dumping', 'uploading', 'completed', 'failed', 'deleting')
|
||||
),
|
||||
constraint backup_records_attempt_ck check (attempt_count >= 0),
|
||||
constraint backup_records_lifecycle_ck check (
|
||||
created_at <= updated_at
|
||||
and (started_at is null or started_at >= created_at)
|
||||
and (completed_at is null or completed_at >= created_at)
|
||||
and (
|
||||
(
|
||||
status = 'queued'
|
||||
and started_at is null
|
||||
and completed_at is null
|
||||
)
|
||||
or (
|
||||
status in ('dumping', 'uploading')
|
||||
and started_at is not null
|
||||
and completed_at is null
|
||||
)
|
||||
or (
|
||||
status in ('completed', 'failed', 'deleting')
|
||||
and started_at is not null
|
||||
and completed_at is not null
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint backup_records_artifact_ck check (
|
||||
(
|
||||
status not in ('uploading', 'completed')
|
||||
or (size_bytes is not null and sha256 is not null)
|
||||
)
|
||||
and (size_bytes is null or size_bytes >= 0)
|
||||
and (sha256 is null or sha256 ~ '^[0-9a-f]{64}$')
|
||||
),
|
||||
constraint backup_records_error_ck check (
|
||||
(error_code is null) = (error_message is null)
|
||||
and (status <> 'failed' or (error_code is not null and error_message is not null))
|
||||
and (
|
||||
status not in ('queued', 'dumping', 'uploading', 'completed')
|
||||
or error_code is null
|
||||
)
|
||||
),
|
||||
-- expires_at 为空,或不早于创建时间(创建时 now + 天数 恒晚于 created_at)。
|
||||
constraint backup_records_expiry_ck check (
|
||||
expires_at is null or expires_at >= created_at
|
||||
)
|
||||
);
|
||||
|
||||
create index backup_records_created_idx
|
||||
on backup_records (created_at desc, id desc);
|
||||
create index backup_records_status_idx
|
||||
on backup_records (status, created_at, id);
|
||||
create unique index backup_records_scheduled_uq
|
||||
on backup_records (scheduled_at)
|
||||
where trigger_kind = 'scheduled' and scheduled_at is not null;
|
||||
create unique index backup_records_active_uq
|
||||
on backup_records ((1))
|
||||
where status in ('queued', 'dumping', 'uploading');
|
||||
create unique index backup_records_object_key_uq
|
||||
on backup_records (object_key)
|
||||
where object_key is not null;
|
||||
-- 同时服务 retentionDays 与 retentionCount。
|
||||
create index backup_records_scheduled_completed_idx
|
||||
on backup_records (completed_at desc, id desc)
|
||||
where trigger_kind = 'scheduled' and status = 'completed';
|
||||
-- 可清理记录(expires_at 已到期且非活跃)的扫描索引。
|
||||
create index backup_records_retention_idx
|
||||
on backup_records (expires_at)
|
||||
where expires_at is not null and status in ('completed', 'failed');
|
||||
@@ -1,5 +0,0 @@
|
||||
-- 删除被主键覆盖的冗余唯一索引(docs/project-redundancy-boundary-audit.md DB-01)。
|
||||
-- provider_accounts.id 已是主键,(id, provider_kind) 第二列不增加唯一性,
|
||||
-- 工程内也没有外键依赖该复合索引。
|
||||
|
||||
drop index provider_accounts_id_kind_uq;
|
||||
@@ -147,7 +147,7 @@ Admin API 不要求客户端提交配置 revision,也不向客户端暴露配
|
||||
|
||||
## 8. PostgreSQL 终态
|
||||
|
||||
`backend/migrations/0001_initial.sql` 创建且只创建七张业务表;`0003_s3_backup.sql` 增加备份配置与备份记录两张表:
|
||||
`backend/migrations/0001_initial.sql` 直接定义当前大版本完整的新库基线,共创建九张业务表:
|
||||
|
||||
| 表 | 权威事实 |
|
||||
| --- | --- |
|
||||
|
||||
@@ -16,9 +16,9 @@
|
||||
"@boxicons/vue": "1.0.1",
|
||||
"@fontsource-variable/inter": "^5.3.0",
|
||||
"@fontsource-variable/jetbrains-mono": "^5.3.0",
|
||||
"@lucide/vue": "^1.27.0",
|
||||
"@vueuse/core": "^14.3.0",
|
||||
"axios": "^1.18.1",
|
||||
"@lucide/vue": "^1.28.0",
|
||||
"@vueuse/core": "^14.4.0",
|
||||
"axios": "^1.19.0",
|
||||
"dayjs": "^1.11.21",
|
||||
"dompurify": "^3.4.12",
|
||||
"echarts": "^6.1.0",
|
||||
@@ -33,7 +33,7 @@
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^9.2.0",
|
||||
"@tailwindcss/vite": "^4.3.3",
|
||||
"@types/node": "^26.1.1",
|
||||
"@types/node": "^26.1.2",
|
||||
"@vitejs/plugin-vue": "^6.0.8",
|
||||
"@vue/tsconfig": "^0.9.1",
|
||||
"eslint": "^10.8.0",
|
||||
@@ -41,7 +41,7 @@
|
||||
"eslint-plugin-vuejs-accessibility": "^2.5.0",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"typescript": "^6.0.3",
|
||||
"vite": "^8.1.5",
|
||||
"vue-tsc": "^3.3.8"
|
||||
"vite": "^8.2.0",
|
||||
"vue-tsc": "^3.3.9"
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+314
-167
File diff suppressed because it is too large
Load Diff
@@ -7,12 +7,16 @@ export interface AccountQuotaWindow {
|
||||
labelDisplay: string
|
||||
usedPercent: number | null
|
||||
usedPercentDisplay: string
|
||||
limitReached: boolean
|
||||
localUsage?: unknown
|
||||
resetAtDisplay: string
|
||||
}
|
||||
|
||||
export interface AccountQuota {
|
||||
refreshedAtDisplay: string
|
||||
limitReached: boolean
|
||||
// 429 临时限流(Redis 冷却)到期时间;非限流中为 null。
|
||||
rateLimitedUntil: string | null
|
||||
windows: AccountQuotaWindow[]
|
||||
}
|
||||
|
||||
@@ -102,7 +106,8 @@ export interface Account {
|
||||
hasRefreshToken: boolean
|
||||
status: string
|
||||
displayStatus: string
|
||||
tokenRefreshing: boolean
|
||||
errorReason: string | null
|
||||
errorMessage: string | null
|
||||
availability: string
|
||||
enabled: boolean
|
||||
accessTokenExpiresAt: string | null
|
||||
@@ -126,9 +131,11 @@ export interface AccountPageMeta {
|
||||
|
||||
export interface AccountSummary {
|
||||
total: number
|
||||
active: number
|
||||
normal: number
|
||||
quotaExhausted: number
|
||||
unavailable: number
|
||||
rateLimited: number
|
||||
disabled: number
|
||||
error: number
|
||||
}
|
||||
|
||||
export interface AccountListResponse {
|
||||
|
||||
@@ -144,7 +144,9 @@ export interface DashboardAccountUsage {
|
||||
export interface DashboardPoolSummary {
|
||||
total: number
|
||||
active: number
|
||||
rateLimited: number
|
||||
expired: number
|
||||
invalid: number
|
||||
quotaExhausted: number
|
||||
refreshing: number | null
|
||||
disabled: number
|
||||
|
||||
@@ -108,11 +108,13 @@ const labelClasses = computed(() => [
|
||||
:title="title || (iconOnly ? label : undefined)"
|
||||
:aria-busy="loading"
|
||||
>
|
||||
<LoaderCircle
|
||||
<span
|
||||
v-if="loading"
|
||||
class="shrink-0 animate-spin origin-center transform-view will-change-transform"
|
||||
:size="loadingIconSize[size]"
|
||||
/>
|
||||
class="inline-flex shrink-0 animate-spin origin-center will-change-transform"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<LoaderCircle :size="loadingIconSize[size]" />
|
||||
</span>
|
||||
<span v-if="$slots.icon && !loading" class="inline-flex shrink-0">
|
||||
<slot name="icon" />
|
||||
</span>
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
|
||||
type CardVariant = 'default' | 'dashboard' | 'elevated'
|
||||
type CardPadding = 'default' | 'compact'
|
||||
type CardLayout = 'content' | 'flush'
|
||||
type HeaderCollapseAt = 'sm' | 'lg' | 'none'
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
as?: keyof HTMLElementTagNameMap
|
||||
padded?: boolean
|
||||
variant?: CardVariant
|
||||
padding?: CardPadding
|
||||
layout?: CardLayout
|
||||
title?: string
|
||||
description?: string
|
||||
headerCollapseAt?: HeaderCollapseAt
|
||||
@@ -17,8 +18,8 @@ const props = withDefaults(
|
||||
}>(),
|
||||
{
|
||||
as: 'section',
|
||||
padded: true,
|
||||
variant: 'default',
|
||||
padding: 'default',
|
||||
layout: 'content',
|
||||
title: undefined,
|
||||
description: undefined,
|
||||
headerCollapseAt: 'sm',
|
||||
@@ -36,16 +37,9 @@ const slots = defineSlots<{
|
||||
default?: () => unknown
|
||||
}>()
|
||||
|
||||
const paddingClasses: Record<CardVariant, string> = {
|
||||
default: 'px-5 py-3',
|
||||
dashboard: 'px-7 py-5.5',
|
||||
elevated: 'px-5 py-3',
|
||||
}
|
||||
|
||||
const shadowClasses: Record<CardVariant, string> = {
|
||||
default: 'shadow-(--cp-shadow-card)',
|
||||
dashboard: 'shadow-(--cp-shadow-card)',
|
||||
elevated: 'shadow-(--cp-shadow-popover)',
|
||||
const paddingClasses: Record<CardPadding, string> = {
|
||||
default: 'p-5.5',
|
||||
compact: 'p-4',
|
||||
}
|
||||
|
||||
const hasManagedHeader = computed(
|
||||
@@ -53,6 +47,12 @@ const hasManagedHeader = computed(
|
||||
!!props.title || !!props.description || !!slots.actions || !!slots.title || !!slots.description,
|
||||
)
|
||||
|
||||
const outerPaddingClass = computed(() => {
|
||||
if (props.layout === 'flush')
|
||||
return undefined
|
||||
return paddingClasses[props.padding]
|
||||
})
|
||||
|
||||
const managedHeaderLayoutClasses = computed(() => {
|
||||
if (props.headerCollapseAt === 'none') {
|
||||
return 'flex items-start justify-between gap-3'
|
||||
@@ -69,14 +69,17 @@ const managedHeaderLayoutClasses = computed(() => {
|
||||
<template>
|
||||
<component
|
||||
:is="props.as"
|
||||
class="[--cp-input-current-bg:var(--cp-input-soft-bg)] [--cp-input-current-bg-hover:var(--cp-input-soft-bg-hover)] overflow-hidden bg-(--cp-bg-surface) rounded-(--cp-card-radius)"
|
||||
class="[--cp-input-current-bg:var(--cp-input-soft-bg)] [--cp-input-current-bg-hover:var(--cp-input-soft-bg-hover)] overflow-hidden rounded-(--cp-card-radius) bg-(--cp-bg-surface) shadow-(--cp-shadow-card)"
|
||||
:class="[
|
||||
shadowClasses[props.variant],
|
||||
props.padded ? paddingClasses[props.variant] : undefined,
|
||||
outerPaddingClass,
|
||||
]"
|
||||
>
|
||||
<template v-if="$slots.header || hasManagedHeader || $slots.body">
|
||||
<header v-if="$slots.header || hasManagedHeader" class="shrink-0" :class="props.headerClass">
|
||||
<header
|
||||
v-if="$slots.header || hasManagedHeader"
|
||||
class="shrink-0"
|
||||
:class="props.headerClass"
|
||||
>
|
||||
<slot name="header">
|
||||
<div :class="managedHeaderLayoutClasses">
|
||||
<div class="min-w-0 pt-0.5">
|
||||
@@ -105,7 +108,10 @@ const managedHeaderLayoutClasses = computed(() => {
|
||||
</slot>
|
||||
</header>
|
||||
|
||||
<div v-if="$slots.body || $slots.default" :class="props.bodyClass">
|
||||
<div
|
||||
v-if="$slots.body || $slots.default"
|
||||
:class="props.bodyClass"
|
||||
>
|
||||
<slot name="body">
|
||||
<slot />
|
||||
</slot>
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
<script setup lang="ts">
|
||||
defineProps<{
|
||||
const props = defineProps<{
|
||||
title: string
|
||||
description?: string
|
||||
actionsInline?: boolean
|
||||
}>()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<header class="grid min-h-17 shrink-0 grid-cols-[minmax(0,1fr)_auto] items-start gap-x-4 gap-y-3">
|
||||
<header
|
||||
class="grid min-h-17 shrink-0 items-start"
|
||||
:class="
|
||||
props.actionsInline
|
||||
? 'grid-cols-[minmax(0,1fr)_auto] gap-x-4'
|
||||
: 'grid-cols-1 gap-y-3 sm:grid-cols-[minmax(0,1fr)_auto] sm:gap-x-4'
|
||||
"
|
||||
>
|
||||
<div class="min-w-0">
|
||||
<h1 class="m-0 text-[34px] leading-[1.15] font-extrabold text-(--cp-text-primary)">
|
||||
{{ title }}
|
||||
@@ -23,6 +31,7 @@ defineProps<{
|
||||
<div
|
||||
v-if="$slots.actions"
|
||||
class="mt-0.5 flex shrink-0 items-center justify-end gap-2"
|
||||
:class="props.actionsInline ? undefined : 'max-sm:w-full'"
|
||||
>
|
||||
<slot name="actions" />
|
||||
</div>
|
||||
|
||||
@@ -29,6 +29,8 @@ const props = withDefaults(
|
||||
offset?: number
|
||||
width?: number | string
|
||||
disabled?: boolean
|
||||
/** hover 触发时打开弹窗的延迟(毫秒);click 触发不生效。 */
|
||||
hoverDelay?: number
|
||||
panelClass?: string
|
||||
triggerClass?: string
|
||||
anchorElement?: HTMLElement | null
|
||||
@@ -39,6 +41,7 @@ const props = withDefaults(
|
||||
trigger: 'click',
|
||||
offset: 6,
|
||||
disabled: false,
|
||||
hoverDelay: 400,
|
||||
panelClass: '',
|
||||
triggerClass: '',
|
||||
anchorElement: null,
|
||||
@@ -55,6 +58,7 @@ const popoverRef = ref<HTMLElement | null>(null)
|
||||
const popoverStyle = ref<CSSProperties>({})
|
||||
const popoverArrowStyle = ref<CSSProperties>({})
|
||||
const hoverCloseTimer = shallowRef<number>()
|
||||
const hoverOpenTimer = shallowRef<number>()
|
||||
const viewportTarget = computed(() => (open.value && typeof window !== 'undefined' ? window : null))
|
||||
|
||||
const popoverClasses = computed(() => [
|
||||
@@ -260,6 +264,7 @@ async function openPopover() {
|
||||
if (props.disabled || open.value)
|
||||
return
|
||||
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
open.value = true
|
||||
await nextTick()
|
||||
@@ -293,13 +298,28 @@ function clearHoverCloseTimer() {
|
||||
hoverCloseTimer.value = undefined
|
||||
}
|
||||
|
||||
function clearHoverOpenTimer() {
|
||||
if (hoverOpenTimer.value === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
window.clearTimeout(hoverOpenTimer.value)
|
||||
hoverOpenTimer.value = undefined
|
||||
}
|
||||
|
||||
function handleHoverEnter() {
|
||||
if (props.trigger !== 'hover') {
|
||||
return
|
||||
}
|
||||
|
||||
clearHoverCloseTimer()
|
||||
void openPopover()
|
||||
// 延迟打开:鼠标稳定悬停一小段时间再展示,避免扫过表格时弹窗频繁闪烁。
|
||||
if (hoverOpenTimer.value === undefined) {
|
||||
hoverOpenTimer.value = window.setTimeout(() => {
|
||||
hoverOpenTimer.value = undefined
|
||||
void openPopover()
|
||||
}, props.hoverDelay)
|
||||
}
|
||||
}
|
||||
|
||||
function handleHoverLeave() {
|
||||
@@ -307,6 +327,7 @@ function handleHoverLeave() {
|
||||
return
|
||||
}
|
||||
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
hoverCloseTimer.value = window.setTimeout(closePopover, 90)
|
||||
}
|
||||
@@ -341,7 +362,10 @@ useEventListener(viewportTarget, 'keydown', (event) => {
|
||||
})
|
||||
useEventListener(viewportTarget, 'resize', updatePopoverPositionThrottled)
|
||||
useEventListener(viewportTarget, 'scroll', updatePopoverPositionThrottled, { capture: true })
|
||||
onBeforeUnmount(clearHoverCloseTimer)
|
||||
onBeforeUnmount(() => {
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
||||
@@ -57,7 +57,12 @@ function connectionLogClass(tone: string) {
|
||||
class="flex items-center justify-between gap-4 rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3"
|
||||
>
|
||||
<AccountIdentityCell :account="account" size="lg" show-plan />
|
||||
<AccountStatusBadge :status="account.status" variant="pill" />
|
||||
<AccountStatusBadge
|
||||
:status="account.status"
|
||||
:error-reason="account.errorReason"
|
||||
:error-message="account.errorMessage"
|
||||
variant="pill"
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section class="rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3">
|
||||
|
||||
@@ -54,10 +54,13 @@ const provider = defineModel<string>('provider', { required: true })
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="flex shrink-0 self-end items-center justify-end gap-2 md:ml-auto">
|
||||
<div
|
||||
class="grid w-full grid-cols-2 gap-2 md:flex md:w-auto md:shrink-0 md:self-end md:items-center md:justify-end md:ml-auto"
|
||||
>
|
||||
<BaseButton
|
||||
v-if="selectedCount > 0"
|
||||
variant="danger"
|
||||
class="w-full whitespace-nowrap md:w-auto"
|
||||
:disabled="batchDeleting"
|
||||
@click="emit('deleteSelected')"
|
||||
>
|
||||
@@ -67,13 +70,19 @@ const provider = defineModel<string>('provider', { required: true })
|
||||
<BaseButton
|
||||
v-if="selectedCount > 0"
|
||||
variant="default"
|
||||
class="w-full whitespace-nowrap md:w-auto"
|
||||
:loading="exportingAccounts"
|
||||
@click="emit('exportSelected')"
|
||||
>
|
||||
<Download class="size-4" />
|
||||
导出选中 ({{ selectedCount }})
|
||||
</BaseButton>
|
||||
<BaseButton variant="primary" @click="emit('create')">
|
||||
<BaseButton
|
||||
variant="primary"
|
||||
class="whitespace-nowrap md:w-auto"
|
||||
:class="selectedCount > 0 ? 'col-span-2 w-full' : 'col-span-2 justify-self-end'"
|
||||
@click="emit('create')"
|
||||
>
|
||||
<Upload class="size-4" />
|
||||
导入账号
|
||||
</BaseButton>
|
||||
|
||||
@@ -20,22 +20,22 @@ const overviewItems = computed(() => [
|
||||
},
|
||||
{
|
||||
label: '正常账号',
|
||||
value: formatCount(props.summary.active),
|
||||
value: formatCount(props.summary.normal),
|
||||
caption: '可参与调度',
|
||||
tone: 'success',
|
||||
icon: ShieldCheck,
|
||||
},
|
||||
{
|
||||
label: '配额耗尽',
|
||||
value: formatCount(props.summary.quotaExhausted),
|
||||
caption: '等待配额恢复',
|
||||
label: '额度受限',
|
||||
value: formatCount((props.summary.quotaExhausted ?? 0) + (props.summary.rateLimited ?? 0)),
|
||||
caption: '等待额度恢复',
|
||||
tone: 'warning',
|
||||
icon: Gauge,
|
||||
},
|
||||
{
|
||||
label: '异常账号',
|
||||
value: formatCount(props.summary.unavailable),
|
||||
caption: '过期 / 禁用 / 封禁',
|
||||
label: '待处理',
|
||||
value: formatCount((props.summary.disabled ?? 0) + (props.summary.error ?? 0)),
|
||||
caption: '已停用 / 错误',
|
||||
tone: 'danger',
|
||||
icon: AlertTriangle,
|
||||
},
|
||||
@@ -65,16 +65,15 @@ function overviewIconClass(tone: string) {
|
||||
v-for="item in overviewItems"
|
||||
:key="item.label"
|
||||
as="article"
|
||||
:padded="false"
|
||||
class="h-24"
|
||||
padding="compact"
|
||||
>
|
||||
<div class="flex h-full items-stretch justify-between gap-3 px-5 py-3">
|
||||
<div class="flex min-w-0 flex-col justify-between">
|
||||
<div class="flex items-stretch justify-between gap-3">
|
||||
<div class="flex min-w-0 flex-col">
|
||||
<p class="m-0 text-[12px] leading-none font-heavy text-(--cp-text-secondary)">
|
||||
{{ item.label }}
|
||||
</p>
|
||||
<strong
|
||||
class="block font-mono text-[26px] leading-none font-extrabold text-(--cp-text-primary)"
|
||||
class="my-2 block font-mono text-[26px] leading-none font-extrabold text-(--cp-text-primary)"
|
||||
>
|
||||
{{ item.value }}
|
||||
</strong>
|
||||
|
||||
@@ -1,20 +1,68 @@
|
||||
<script setup lang="ts">
|
||||
import { AlertTriangle, CircleCheck, Gauge, Power, Timer } from '@lucide/vue'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import { statusLabels, statusTones } from '../constants'
|
||||
import BasePopover from '@/components/base/BasePopover.vue'
|
||||
import { errorReasonLabels, statusLabels, statusTones } from '../constants'
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
status: string
|
||||
/** `status === 'error'` 时的具体原因(对应后端 `errorReason`)。 */
|
||||
errorReason?: string | null
|
||||
/** 最近一次失败的上游错误原文(对应后端 `errorMessage`),作为辅助证据展示。 */
|
||||
errorMessage?: string | null
|
||||
// 429 临时限流到期时间(对应后端 `quota.rateLimitedUntil`);`status === 'rate_limited'` 时有值。
|
||||
rateLimitedUntil?: string | null
|
||||
variant?: 'inline' | 'pill'
|
||||
}>(),
|
||||
{
|
||||
errorReason: null,
|
||||
errorMessage: null,
|
||||
rateLimitedUntil: null,
|
||||
variant: 'inline',
|
||||
},
|
||||
)
|
||||
|
||||
const tone = computed(() => statusTones[props.status])
|
||||
const label = computed(() => statusLabels[props.status] || props.status)
|
||||
/** 分类原因(受控枚举映射文案);`status === 'error'` 时有值。 */
|
||||
const reasonLabel = computed(() => {
|
||||
if (props.errorReason)
|
||||
return errorReasonLabels[props.errorReason] || props.errorReason
|
||||
return null
|
||||
})
|
||||
/** 上游错误原文(辅助证据);有则显、无则隐。 */
|
||||
const errorText = computed(() => props.errorMessage || null)
|
||||
// 429 临时限流中且带到期时间。
|
||||
const hasRateLimit = computed(() =>
|
||||
props.status === 'rate_limited' && Boolean(props.rateLimitedUntil),
|
||||
)
|
||||
/** 限流到期相对时间:如「剩余 12 分钟」。到点返回 null。 */
|
||||
const rateLimitedRelative = computed(() => {
|
||||
if (!hasRateLimit.value)
|
||||
return null
|
||||
const until = new Date(props.rateLimitedUntil!.replace(' ', 'T')).getTime()
|
||||
const diffMinutes = Math.round((until - Date.now()) / 60_000)
|
||||
if (Number.isNaN(diffMinutes) || diffMinutes < 1)
|
||||
return null
|
||||
if (diffMinutes < 60)
|
||||
return `剩余 ${diffMinutes} 分钟`
|
||||
const hours = Math.floor(diffMinutes / 60)
|
||||
const mins = diffMinutes % 60
|
||||
if (mins === 0)
|
||||
return `剩余 ${hours} 小时`
|
||||
return `剩余 ${hours} 小时 ${mins} 分`
|
||||
})
|
||||
// `error` 状态始终显示弹窗(即使 reason/message 为空也展示状态卡片),
|
||||
// `rate_limited` 需要带到期时间才显示;其余状态不弹窗。
|
||||
const hasDetail = computed(() =>
|
||||
props.status === 'error' || hasRateLimit.value,
|
||||
)
|
||||
/** 弹窗明细区是否渲染(错误原因 / 限流到期 / 错误反馈至少一项)。 */
|
||||
const hasDetailRows = computed(() =>
|
||||
hasRateLimit.value || Boolean(reasonLabel.value) || Boolean(errorText.value),
|
||||
)
|
||||
|
||||
const textClass = computed(() => {
|
||||
if (tone.value === 'success') {
|
||||
@@ -47,22 +95,99 @@ const dotClass = computed(() => {
|
||||
}
|
||||
return 'bg-(--cp-text-muted)'
|
||||
})
|
||||
|
||||
/** 状态卡片左侧图标块:状态色 soft-bg + 语义图标。 */
|
||||
const cardIcon = computed(() => {
|
||||
switch (props.status) {
|
||||
case 'error': return AlertTriangle
|
||||
case 'rate_limited': return Timer
|
||||
case 'quota_exhausted': return Gauge
|
||||
case 'disabled': return Power
|
||||
default: return CircleCheck
|
||||
}
|
||||
})
|
||||
const iconBlockClass = computed(() => {
|
||||
if (tone.value === 'success') {
|
||||
return 'bg-(--cp-success-bg) text-(--cp-success-text)'
|
||||
}
|
||||
if (tone.value === 'danger') {
|
||||
return 'bg-(--cp-danger-bg) text-(--cp-danger-text)'
|
||||
}
|
||||
if (tone.value === 'warning') {
|
||||
return 'bg-(--cp-warning-bg) text-(--cp-warning-text)'
|
||||
}
|
||||
if (tone.value === 'info') {
|
||||
return 'bg-(--cp-info-bg) text-(--cp-info-text)'
|
||||
}
|
||||
return 'bg-(--cp-bg-subtle) text-(--cp-text-secondary)'
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<span
|
||||
v-if="variant === 'pill'"
|
||||
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] font-heavy"
|
||||
:class="textClass"
|
||||
<BasePopover
|
||||
:disabled="!hasDetail"
|
||||
trigger="hover"
|
||||
placement="top-start"
|
||||
:width="240"
|
||||
panel-class="!p-3"
|
||||
>
|
||||
{{ label }}
|
||||
</span>
|
||||
<span
|
||||
v-else
|
||||
class="inline-flex min-w-16 items-center gap-1.5 text-[12px] leading-none font-emphasis"
|
||||
:class="textClass"
|
||||
>
|
||||
<span class="size-1.5 rounded-full" :class="dotClass" />
|
||||
<span>{{ label }}</span>
|
||||
</span>
|
||||
<template #trigger>
|
||||
<span
|
||||
v-if="variant === 'pill'"
|
||||
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] font-heavy"
|
||||
:class="textClass"
|
||||
>
|
||||
{{ label }}
|
||||
</span>
|
||||
<span
|
||||
v-else
|
||||
class="inline-flex min-w-16 items-center gap-1.5 text-[12px] leading-none font-emphasis"
|
||||
:class="textClass"
|
||||
>
|
||||
<span class="size-1.5 rounded-full" :class="dotClass" />
|
||||
<span>{{ label }}</span>
|
||||
</span>
|
||||
</template>
|
||||
|
||||
<div
|
||||
class="flex gap-3 text-[12px] leading-none"
|
||||
:class="hasDetailRows ? 'items-start' : 'items-center'"
|
||||
>
|
||||
<!-- 状态图标块:soft 状态色背景圆角块 -->
|
||||
<span
|
||||
class="inline-flex size-8 shrink-0 items-center justify-center rounded-(--cp-icon-button-radius)"
|
||||
:class="iconBlockClass"
|
||||
>
|
||||
<component :is="cardIcon" class="size-4" />
|
||||
</span>
|
||||
|
||||
<div class="min-w-0 flex-1">
|
||||
<!-- 状态标题行:只显示状态名 -->
|
||||
<div class="font-heavy text-(--cp-text-primary)">
|
||||
{{ label }}
|
||||
</div>
|
||||
|
||||
<div v-if="hasDetailRows" class="mt-2 border-t border-(--cp-divider-subtle) pt-2">
|
||||
<!-- 错误:具体原因独立成行 -->
|
||||
<div v-if="reasonLabel" class="leading-4 text-(--cp-text-secondary)">
|
||||
{{ reasonLabel }}
|
||||
</div>
|
||||
<!-- 限流中:显示相对到期时间 -->
|
||||
<div v-if="hasRateLimit" class="flex items-baseline gap-2">
|
||||
<span class="w-20 shrink-0 font-semibold text-(--cp-text-tertiary)">限流到期</span>
|
||||
<span class="font-medium text-(--cp-text-primary)">
|
||||
{{ rateLimitedRelative ?? props.rateLimitedUntil }}
|
||||
</span>
|
||||
</div>
|
||||
<!-- 错误:显示上游反馈原文 -->
|
||||
<div v-if="errorText" class="mt-1 flex items-start gap-2">
|
||||
<span class="w-14 shrink-0 pt-px font-semibold leading-3.5 text-(--cp-text-tertiary)">上游反馈</span>
|
||||
<span class="min-w-0 flex-1 wrap-break-word leading-3.5 text-(--cp-text-secondary)">
|
||||
{{ errorText }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</BasePopover>
|
||||
</template>
|
||||
|
||||
@@ -26,7 +26,7 @@ const emit = defineEmits<{
|
||||
const canRefreshToken = computed(
|
||||
() =>
|
||||
props.account.hasRefreshToken
|
||||
&& (props.account.status === 'active' || props.account.status === 'quota_exhausted'),
|
||||
&& (props.account.status === 'normal' || props.account.status === 'quota_exhausted'),
|
||||
)
|
||||
const canReauthorize = computed(() => props.account.provider === 'openai')
|
||||
</script>
|
||||
|
||||
@@ -230,7 +230,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
testingAccount.value = {
|
||||
...account,
|
||||
status,
|
||||
displayStatus: account.tokenRefreshing ? account.displayStatus : status,
|
||||
displayStatus: status,
|
||||
}
|
||||
options.onAccountStatus(account.id, status)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user