mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-13 08:06:41 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
  ZyyandCodex
|
12c32fcdec | ||
 Ke WangandCodex
|
26de671226 | ||
|
Ke WangandCodex
|
a92b95c48c | ||
|
ZyyandCodex
|
e7c24781fb | ||
|
ZyyandCodex
|
3b4b547e9c | ||
|
ZyyandCodex
|
2016bda031 | ||
|
ZyyandCodex
|
790f3147e0 | ||
|
ZyyandCodex
|
96e8443ede | ||
|
ZyyandCodex
|
42badf64ca | ||
|
35afe1123f | ||
|
ZyyandCodex
|
2ac4265832 | ||
|
ZyyandCodex
|
d22ec29824 | ||
|
ZyyandCodex
|
7a32c6cec2 | ||
|
|
0ff1a72919 | ||
|
|
79083a5632 | ||
|
ZyyandCodex
|
90f7456db8 | ||
|
ZyyandCodex
|
6f71b27c3c | ||
|
ZyyandCodex
|
7b05b14a2b | ||
|
f9b5b5ff77 | ||
|
dependabot[bot]
|
4d5778a84f | ||
|
dependabot[bot]
|
6d22d73506 | ||
|
dependabot[bot]
|
d8ab207282 | ||
|
dependabot[bot]
|
3816b46ba1 | ||
|
dependabot[bot]
|
4a7c06f895 |
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -171,6 +171,12 @@ jobs:
|
||||
toolchain: ${{ needs.release-info.outputs.rust }}
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Configure Windows runtime linkage
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
# 发行包静态链接 MSVC 运行库，解压运行时无需另外安装 VC++ Redistributable
|
||||
run: echo 'RUSTFLAGS=-C target-feature=+crt-static' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
@@ -186,6 +192,8 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cargo build --release --locked --target "${{ matrix.target }}" --bin codex-proxy-rs
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --help
|
||||
"target/${{ matrix.target }}/release/${{ matrix.binary }}" --version
|
||||
mkdir -p "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}"
|
||||
cp "target/${{ matrix.target }}/release/${{ matrix.binary }}" "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/${{ matrix.binary }}"
|
||||
working-directory: backend
|
||||
@@ -195,6 +203,16 @@ jobs:
|
||||
CPR_BUILD_TIME: ${{ needs.release-info.outputs.build_time }}
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Verify Windows platform behavior
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
working-directory: backend
|
||||
env:
|
||||
RUST_MIN_STACK: '16777216'
|
||||
run: |
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-admin --test main use_case::plugins::official::
|
||||
cargo test --release --locked --target "${{ matrix.target }}" -p gateway-host --test main system_update::
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
@@ -40,6 +40,8 @@
|
||||
|
||||
以下命令适用于 Linux amd64/arm64，需要 Docker Engine、Docker Compose Plugin、curl 和 OpenSSL
|
||||
|
||||
Windows x64 / ARM64 使用发行包部署，步骤见 [Windows 二进制部署](deploy/README.md#windows-二进制部署)
|
||||
|
||||
已有部署请先看 [升级说明](deploy/README.md#镜像升级与源码构建)，不要覆盖原配置
|
||||
|
||||
### 一键安装
|
||||
|
||||
Generated
+40
-38
@@ -286,9 +286,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.155.0"
|
||||
version = "1.156.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a542f0e9261b9ce56708d591a62062079fa3c58a362ffbf49a2e48a4f6ced16"
|
||||
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.3"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16"
|
||||
checksum = "f385da9b7757ed4c8214797ad663ff534fb622006143530aa80e21f8e06ff85c"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -387,9 +387,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.64.2"
|
||||
version = "0.64.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398"
|
||||
checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -408,9 +408,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.5.1"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e"
|
||||
checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -432,9 +432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.65.0"
|
||||
version = "0.65.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666"
|
||||
checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -452,9 +452,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a"
|
||||
checksum = "4cb80fb19c084da5fca9d797845735e184445e5b958599440d67529bf6ef1b64"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -475,9 +475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.1"
|
||||
version = "1.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4"
|
||||
checksum = "8c4e85cffe9944b10d576b075432bd8f98dbc081e8681cb37d5b2d87dc27bc84"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -492,9 +492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
|
||||
checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -514,9 +514,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822"
|
||||
checksum = "cf183d70774ea2cfcc101ca74a8884e4c76be8b9df65cd303f6a1175c88e8f80"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -537,9 +537,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.64.0"
|
||||
version = "0.64.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413"
|
||||
checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -2485,9 +2485,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.60.0"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e7305938cac5ee144352ceb71583ffc6cfeac7d20fcb20ef865a4d5c314e499"
|
||||
checksum = "a42a11b47492e6a2cc76e9c7bcc8077776cb31dac444e0795d7f61bcc6441c08"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2513,18 +2513,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8a848331614144c03e815fb931f238c9e342bd7ab501b29e35d647e7cd5688"
|
||||
checksum = "107aeb6ff14b8deb5610096a8ce5c554ec89806b521147eaeb3cae05b07edb7a"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5a947ef798eeacacb43edf29b1d0ab64bdf5e2c9fbcaad10ee8663d922b544"
|
||||
checksum = "e281d6b177bb4834f2ffd6f92aa0443cffe29c996b251c8d94eb6f456186feba"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -3578,9 +3578,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.1"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -3634,9 +3634,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.60.1"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "922268812b6f9e20b4ab0f1efe8e07036958eed7346ceddb31c777b5531fd11f"
|
||||
checksum = "f42818036d3b37cda6bd4c494543158a36e431cb717015ef7c93acc6ba214d3b"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3759,9 +3759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.21.1"
|
||||
version = "0.21.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
|
||||
checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -4953,9 +4953,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
@@ -5052,9 +5052,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.28.0"
|
||||
version = "1.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -5646,16 +5646,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.5"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"nix",
|
||||
"typed-path",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! 网关命令行入口，选择启动模式并创建异步运行时
|
||||
|
||||
use std::{
|
||||
env,
|
||||
error::Error,
|
||||
io::{self, Write},
|
||||
process::ExitCode,
|
||||
@@ -21,7 +20,7 @@ fn main() -> ExitCode {
|
||||
|
||||
fn run() -> Result<ExitCode, Box<dyn Error + Send + Sync>> {
|
||||
ensure_rustls_provider();
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
let mut arguments = gateway_host::system_update::startup_arguments()?;
|
||||
let command = arguments
|
||||
.next()
|
||||
.map(|value| {
|
||||
|
||||
@@ -739,6 +739,8 @@ pub struct UsageListRecord {
|
||||
/// 请求选号时冻结的账号套餐
|
||||
pub provider_account_plan_type: Option<String>,
|
||||
pub provider_account_plan_type_display: Option<String>,
|
||||
/// 查询时的互联识别结果，缺少当前摘要时为未知
|
||||
pub provider_account_upstream_peer: Option<bool>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
@@ -1439,6 +1441,8 @@ pub struct DiagnosticsItem {
|
||||
pub name: String,
|
||||
pub account_authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
/// 仅账号维度补充当前互联识别结果
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
|
||||
@@ -165,8 +165,9 @@ pub struct ConfigurePluginInstance {
|
||||
pub artifact_sha256: String,
|
||||
pub enabled: bool,
|
||||
pub configuration: serde_json::Value,
|
||||
/// 缺省保留旧值；显式空对象清除全部 secret
|
||||
pub secrets: Option<BTreeMap<String, SecretString>>,
|
||||
/// 按字段更新，省略的字段保留，null 清除对应字段
|
||||
#[serde(default)]
|
||||
pub secrets: BTreeMap<String, Option<SecretString>>,
|
||||
pub bindings: Vec<PluginCapabilityBinding>,
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
|
||||
use crate::{
|
||||
@@ -133,7 +134,8 @@ impl DefaultObservabilityService {
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
let runtime_slots = observation.runtime_slots;
|
||||
self.enrich_list_records(&mut observation.recent_requests);
|
||||
self.enrich_list_records(&mut observation.recent_requests)
|
||||
.await;
|
||||
self.enrich_dashboard_quotas(&mut observation.account_usage)
|
||||
.await;
|
||||
for account in &mut observation.account_usage {
|
||||
@@ -259,7 +261,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.list_usage_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage records"))?;
|
||||
self.enrich_list_records(&mut page.items);
|
||||
self.enrich_list_records(&mut page.items).await;
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -336,6 +338,12 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
.usage_diagnostics(range, filter, dimension, 100)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage diagnostics"))?;
|
||||
let peers = if dimension == DiagnosticDimension::Account {
|
||||
self.current_upstream_peers(observation.items.iter().map(|item| item.key.as_str()))
|
||||
.await
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let items = observation
|
||||
.items
|
||||
.into_iter()
|
||||
@@ -345,6 +353,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
rate_or_zero(item.non_completion_count, item.request_count);
|
||||
let retry_rate = rate_or_zero(item.retried_request_count, item.request_count);
|
||||
DiagnosticsItem {
|
||||
account_upstream_peer: peers.get(&item.key).copied(),
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
account_authentication_kind: item.account_authentication_kind,
|
||||
@@ -912,6 +921,29 @@ fn usd_cost(costs: &[CurrencyCost]) -> Option<DecimalAmount> {
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
async fn current_upstream_peers<'a>(
|
||||
&self,
|
||||
ids: impl Iterator<Item = &'a str>,
|
||||
) -> HashMap<String, bool> {
|
||||
let Some(peer) = self.upstream_info.as_deref() else {
|
||||
return HashMap::new();
|
||||
};
|
||||
let ids = ids
|
||||
.map(str::to_owned)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
// 同一页重复账号只读取一次摘要，单个账号不存在或读取失败不阻断历史记录
|
||||
futures::stream::iter(ids)
|
||||
.map(|id: String| async move {
|
||||
let account_id = ProviderAccountId::new(id.clone()).ok()?;
|
||||
let upstream = peer.read_summary(&account_id).await.ok()??.upstream?;
|
||||
Some((id, upstream.discovered && upstream.observed_at.is_some()))
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.filter_map(|value| async move { value })
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn enrich_dashboard_quotas(&self, accounts: &mut [DashboardAccountUsage]) {
|
||||
if accounts.is_empty() {
|
||||
return;
|
||||
@@ -976,8 +1008,22 @@ impl DefaultObservabilityService {
|
||||
|
||||
/// 补充当前套餐展示名称，逐条尽力把可校验的总额升级为完整分解；单条脏数据（非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败）只保留该条已存的总额，不影响整页返回
|
||||
fn enrich_list_records(&self, records: &mut [crate::model::observability::UsageListRecord]) {
|
||||
async fn enrich_list_records(
|
||||
&self,
|
||||
records: &mut [crate::model::observability::UsageListRecord],
|
||||
) {
|
||||
let peers = self
|
||||
.current_upstream_peers(
|
||||
records
|
||||
.iter()
|
||||
.filter_map(|record| record.provider_account_ref.as_deref()),
|
||||
)
|
||||
.await;
|
||||
for record in records {
|
||||
record.provider_account_upstream_peer = record
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.and_then(|id| peers.get(id).copied());
|
||||
record.provider_account_plan_type_display =
|
||||
record.provider_kind.as_deref().and_then(|provider| {
|
||||
self.providers
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
peer_link::{PeerLinkExport, PeerLinkLimits},
|
||||
provider_credentials::ProviderQuotaReadRequest,
|
||||
provider_credentials::{ProviderQuotaReadRequest, explicit_plan_type},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
@@ -175,10 +175,16 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "peer account runtime"))?;
|
||||
let mut quotas = BTreeMap::new();
|
||||
if sharing.share_quota {
|
||||
if sharing.share_quota || sharing.share_plans {
|
||||
let mut window_count = 0usize;
|
||||
let mut by_provider = BTreeMap::new();
|
||||
for account in &accounts {
|
||||
// 仅公开套餐时只补读缺失套餐的本地额度，不回源刷新或扩大公开范围
|
||||
if !sharing.share_quota
|
||||
&& explicit_plan_type(account.plan_type.as_deref()).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
by_provider
|
||||
.entry(&account.provider_kind)
|
||||
.or_insert_with(Vec::new)
|
||||
@@ -200,9 +206,11 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
.zip(provider.read_quotas(&requests).await)
|
||||
{
|
||||
if let Ok(quota) = quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
if sharing.share_quota {
|
||||
window_count = window_count.saturating_add(quota.windows.len());
|
||||
if window_count > limits.max_quota_windows {
|
||||
return Err(PeerLinkError::SnapshotTooLarge);
|
||||
}
|
||||
}
|
||||
quotas.insert(account.id.clone(), quota);
|
||||
}
|
||||
@@ -215,6 +223,7 @@ impl PeerLinkService for DefaultPeerLinkService {
|
||||
if sharing.share_pool {
|
||||
result.pool = Some(projection::pool_summary(
|
||||
&accounts,
|
||||
"as,
|
||||
&runtime,
|
||||
&self.providers,
|
||||
sharing.share_plans,
|
||||
|
||||
@@ -25,6 +25,7 @@ use crate::{
|
||||
|
||||
pub(super) fn pool_summary(
|
||||
accounts: &[AccountRecord],
|
||||
quotas: &BTreeMap<String, ProviderQuota>,
|
||||
runtime: &AccountRuntimeSnapshot,
|
||||
providers: &ProviderAdminRegistry,
|
||||
share_plans: bool,
|
||||
@@ -43,12 +44,13 @@ pub(super) fn pool_summary(
|
||||
AccountStatus::Error => summary.error += 1,
|
||||
}
|
||||
if let Some(plans) = &mut summary.plans
|
||||
&& let (Some(plan), Ok(provider)) = (
|
||||
&account.plan_type,
|
||||
providers.require(&account.provider_kind),
|
||||
&& let Some(plan) = providers.resolve_account_plan(
|
||||
account.provider_kind.as_str(),
|
||||
&mut account.plan_type.clone(),
|
||||
quotas.get(&account.id),
|
||||
)
|
||||
{
|
||||
*plans.entry(provider.plan_type_display(plan)).or_default() += 1;
|
||||
*plans.entry(plan).or_default() += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
|
||||
@@ -217,7 +217,7 @@ impl PluginsService {
|
||||
artifact_sha256: metadata.sha256.clone(),
|
||||
enabled: !configuration_required && compatible,
|
||||
configuration: configuration.clone(),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: bindings.clone(),
|
||||
};
|
||||
match self.configure_instance(None, input, context).await {
|
||||
|
||||
@@ -28,17 +28,16 @@ pub(super) fn same_creation_request(
|
||||
existing: &PluginInstance,
|
||||
input: &ConfigurePluginInstance,
|
||||
) -> bool {
|
||||
let same_secrets = input
|
||||
let mut requested_secrets = input
|
||||
.secrets
|
||||
.as_ref()
|
||||
.map_or(existing.secrets.is_empty(), |secrets| {
|
||||
secrets.len() == existing.secrets.len()
|
||||
&& secrets.iter().all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
})
|
||||
.iter()
|
||||
.filter_map(|(key, value)| value.as_ref().map(|value| (key, value)));
|
||||
let same_secrets = requested_secrets.clone().count() == existing.secrets.len()
|
||||
&& requested_secrets.all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
});
|
||||
existing.name == input.name.trim()
|
||||
&& existing.artifact_sha256 == input.artifact_sha256
|
||||
|
||||
@@ -158,7 +158,6 @@ impl PluginsService {
|
||||
}
|
||||
let saved = if let Some(previous) = &existing
|
||||
&& previous.artifact_sha256 != input.artifact_sha256
|
||||
&& input.secrets.is_none()
|
||||
{
|
||||
self.store
|
||||
.load_version_configuration(&previous.id, &input.artifact_sha256)
|
||||
@@ -167,6 +166,21 @@ impl PluginsService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut secrets = saved.map(|saved| saved.secrets).unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
});
|
||||
for (name, value) in input.secrets {
|
||||
match value {
|
||||
Some(value) => {
|
||||
secrets.insert(name, value);
|
||||
}
|
||||
None => {
|
||||
secrets.remove(&name);
|
||||
}
|
||||
}
|
||||
}
|
||||
let instance = PluginInstance {
|
||||
id: existing.as_ref().map_or_else(
|
||||
|| {
|
||||
@@ -182,14 +196,7 @@ impl PluginsService {
|
||||
enabled: input.enabled,
|
||||
trusted_process: true,
|
||||
configuration: input.configuration,
|
||||
secrets: input
|
||||
.secrets
|
||||
.or_else(|| saved.map(|saved| saved.secrets))
|
||||
.unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
}),
|
||||
secrets,
|
||||
bindings: input.bindings,
|
||||
revision: candidate_revision,
|
||||
};
|
||||
|
||||
@@ -366,6 +366,7 @@ fn target_environment_matches(target: &str) -> bool {
|
||||
match target {
|
||||
"x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => cfg!(target_env = "gnu"),
|
||||
"aarch64-apple-darwin" => cfg!(target_os = "macos"),
|
||||
"x86_64-pc-windows-msvc" | "aarch64-pc-windows-msvc" => cfg!(target_env = "msvc"),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -392,4 +393,18 @@ const SUPPORTED_PLATFORMS: &[Platform] = &[
|
||||
runtime_os: "macos",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "amd64",
|
||||
target: "x86_64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "x86_64",
|
||||
},
|
||||
Platform {
|
||||
release_os: "windows",
|
||||
release_architecture: "arm64",
|
||||
target: "aarch64-pc-windows-msvc",
|
||||
runtime_os: "windows",
|
||||
runtime_architecture: "aarch64",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -69,7 +69,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: saved.configuration,
|
||||
secrets: Some(saved.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: saved.bindings,
|
||||
};
|
||||
// 沿用同一份快照的 CAS 和状态迁移流程，不能在检查目标后重新读取并覆盖并发配置
|
||||
@@ -202,7 +202,7 @@ impl PluginsService {
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: existing.enabled,
|
||||
configuration: configuration.configuration,
|
||||
secrets: Some(configuration.secrets),
|
||||
secrets: Default::default(),
|
||||
bindings: configuration.bindings,
|
||||
};
|
||||
// 同一次读取的 CAS 覆盖草稿计算、准备与提交，不能覆盖并发设置
|
||||
|
||||
@@ -1166,6 +1166,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -41,6 +41,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -1384,6 +1384,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -1478,6 +1479,7 @@ fn total_record(
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
//! 实例敏感配置的逐字段更新、版本基准与提交隔离测试
|
||||
|
||||
use super::*;
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
fn secret_values(instance: &PluginInstance) -> serde_json::Value {
|
||||
serde_json::Value::Object(
|
||||
instance
|
||||
.secrets
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), json!(value.expose_secret())))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn secret_changes_apply_to_current_or_target_version_configuration() {
|
||||
for (target, saved_version) in [
|
||||
(OLD_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, false),
|
||||
(NEW_ARTIFACT, true),
|
||||
] {
|
||||
let token = if saved_version {
|
||||
"saved-token"
|
||||
} else {
|
||||
"current-token"
|
||||
};
|
||||
let untouched = if saved_version {
|
||||
"saved-value"
|
||||
} else {
|
||||
"current-value"
|
||||
};
|
||||
for (changes, expected) in [
|
||||
(json!({}), json!({"token":token,"untouched":untouched})),
|
||||
(
|
||||
json!({"adminKey":"new-key"}),
|
||||
json!({"token":token,"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
(
|
||||
json!({"token":"new-token"}),
|
||||
json!({"token":"new-token","untouched":untouched}),
|
||||
),
|
||||
(json!({"token":null}), json!({"untouched":untouched})),
|
||||
(
|
||||
json!({"token":null,"adminKey":"new-key","absent":null}),
|
||||
json!({"untouched":untouched,"adminKey":"new-key"}),
|
||||
),
|
||||
] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
{
|
||||
let mut data = fixture.data.lock().unwrap();
|
||||
data.snapshot.instances[0].secrets = BTreeMap::from([
|
||||
("token".into(), "current-token".into()),
|
||||
("untouched".into(), "current-value".into()),
|
||||
]);
|
||||
if saved_version {
|
||||
data.history.get_mut(NEW_ARTIFACT).unwrap().secrets = BTreeMap::from([
|
||||
("token".into(), "saved-token".into()),
|
||||
("untouched".into(), "saved-value".into()),
|
||||
]);
|
||||
} else {
|
||||
data.history.clear();
|
||||
}
|
||||
}
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = target.into();
|
||||
request.expected_revision = Some(10);
|
||||
request.secrets = serde_json::from_value(changes).unwrap();
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(secret_values(&result.instance), expected);
|
||||
assert_eq!(secret_values(&fixture.snapshot().instances[0]), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_secret_changes_leave_saved_values_unchanged() {
|
||||
for failure in ["stale", "preparation", "size"] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].secrets =
|
||||
BTreeMap::from([("token".into(), "saved-token".into())]);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
request.expected_revision = Some(if failure == "stale" { 9 } else { 10 });
|
||||
request.secrets = BTreeMap::from([
|
||||
("token".into(), None),
|
||||
(
|
||||
"adminKey".into(),
|
||||
Some(
|
||||
if failure == "size" {
|
||||
"x".repeat(8193)
|
||||
} else {
|
||||
"new-key".into()
|
||||
}
|
||||
.into(),
|
||||
),
|
||||
),
|
||||
]);
|
||||
fixture.data.lock().unwrap().fail_preparation = failure == "preparation";
|
||||
let result = service(fixture.clone(), Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await;
|
||||
assert!(result.is_err(), "{failure} must reject the update");
|
||||
assert_eq!(
|
||||
secret_values(&fixture.snapshot().instances[0]),
|
||||
secret_values(&before.instances[0])
|
||||
);
|
||||
assert!(fixture.data.lock().unwrap().saves.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ use serde_json::json;
|
||||
|
||||
use super::TestPluginPorts;
|
||||
|
||||
mod configuration;
|
||||
mod queries;
|
||||
|
||||
const OLD_ARTIFACT: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
@@ -759,7 +760,7 @@ fn input() -> ConfigurePluginInstance {
|
||||
artifact_sha256: NEW_ARTIFACT.into(),
|
||||
enabled: true,
|
||||
configuration: json!({}),
|
||||
secrets: None,
|
||||
secrets: Default::default(),
|
||||
bindings: vec![],
|
||||
}
|
||||
}
|
||||
@@ -942,6 +943,8 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
// 此用例验证创建重试，使用不需要私有状态迁移的固定版本
|
||||
value.artifact_sha256 = OLD_ARTIFACT.into();
|
||||
value.enabled = false;
|
||||
value.secrets =
|
||||
serde_json::from_value(json!({"token": "saved-token", "absent": null})).unwrap();
|
||||
value
|
||||
};
|
||||
let created = service
|
||||
@@ -955,6 +958,12 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
assert_eq!(created.instance.id, creation_id);
|
||||
assert_eq!(retried.instance.id, creation_id);
|
||||
assert_eq!(fixture.snapshot().instances.len(), 2);
|
||||
use secrecy::ExposeSecret as _;
|
||||
assert_eq!(
|
||||
retried.instance.secrets["token"].expose_secret(),
|
||||
"saved-token"
|
||||
);
|
||||
assert!(!retried.instance.secrets.contains_key("absent"));
|
||||
let before = fixture.snapshot().config_revision;
|
||||
let mut changed = request();
|
||||
changed.name = "changed draft".into();
|
||||
@@ -965,6 +974,17 @@ async fn configuration_retry_reuses_creation_id_and_rejects_changed_draft() {
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
let mut changed = request();
|
||||
changed
|
||||
.secrets
|
||||
.insert("token".into(), Some("different-token".into()));
|
||||
let error = service
|
||||
.configure_instance(None, changed, &context())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert_eq!(error.kind(), AdminErrorKind::Conflict);
|
||||
assert_eq!(fixture.snapshot().config_revision, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1435,23 +1455,6 @@ async fn version_switch_preparation_failure_preserves_current_and_saved_settings
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_edit_preserves_the_target_versions_secrets_when_omitted() {
|
||||
use secrecy::ExposeSecret as _;
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let before = fixture.snapshot();
|
||||
let mut request = input();
|
||||
request.secrets = None;
|
||||
let result = service(fixture, Arc::new(Published::default()))
|
||||
.configure_instance(Some(&before.instances[0].id), request, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.instance.secrets["token"].expose_secret(),
|
||||
"old-version-secret"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_requires_a_saved_configuration_instead_of_reusing_incompatible_current_settings()
|
||||
{
|
||||
|
||||
@@ -446,6 +446,8 @@ fn release_platform() -> (&'static str, &'static str, &'static str) {
|
||||
("linux", "x86_64") => ("linux", "amd64", "x86_64-unknown-linux-gnu"),
|
||||
("linux", "aarch64") => ("linux", "arm64", "aarch64-unknown-linux-gnu"),
|
||||
("macos", "aarch64") => ("darwin", "arm64", "aarch64-apple-darwin"),
|
||||
("windows", "x86_64") => ("windows", "amd64", "x86_64-pc-windows-msvc"),
|
||||
("windows", "aarch64") => ("windows", "arm64", "aarch64-pc-windows-msvc"),
|
||||
platform => panic!("unsupported test platform: {platform:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +334,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -271,6 +271,7 @@ pub(crate) fn usage_list_record_view(
|
||||
account_notes: record.provider_account_notes,
|
||||
account_plan_type: record.provider_account_plan_type,
|
||||
account_plan_type_display: record.provider_account_plan_type_display,
|
||||
account_upstream_peer: record.provider_account_upstream_peer,
|
||||
route: record.endpoint,
|
||||
model,
|
||||
requested_model: record.requested_model_id,
|
||||
@@ -1351,6 +1352,7 @@ pub(crate) fn diagnostics_view(
|
||||
request_count: item.request_count,
|
||||
account_plan_type: item.account_plan_type,
|
||||
account_plan_type_display: item.account_plan_type_display,
|
||||
account_upstream_peer: item.account_upstream_peer,
|
||||
success_count: item.success_count,
|
||||
error_count: item.error_count,
|
||||
error_rate: item.error_rate,
|
||||
|
||||
@@ -99,6 +99,7 @@ pub struct UsageListRecordView {
|
||||
pub account_notes: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
@@ -849,6 +850,7 @@ pub struct DiagnosticItemView {
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_plan_type: Option<String>,
|
||||
pub account_plan_type_display: Option<String>,
|
||||
pub account_upstream_peer: Option<bool>,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub error_count: u64,
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -751,6 +752,7 @@ impl RuntimeSettingsFields {
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: self.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: self.openai_account_affinity,
|
||||
max_account_rotations: self.max_account_rotations,
|
||||
@@ -790,6 +792,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: settings.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
|
||||
@@ -618,8 +618,6 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-account-routing-override"
|
||||
| "x-openai-fedramp"
|
||||
// 上游指纹必须由运行时画像统一生成，客户端 originator/User-Agent/version
|
||||
// 不能作为不透明头透传覆盖，避免不同下游客户端暴露不一致的设备指纹
|
||||
| "originator"
|
||||
|
||||
@@ -651,6 +651,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
max_waiting_per_key: command.values.max_waiting_per_key,
|
||||
max_waiting_per_account: command.values.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.values.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: command.values.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: command
|
||||
.values
|
||||
.openai_guardian_reserved_concurrency,
|
||||
@@ -1893,6 +1894,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -656,6 +656,23 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
);
|
||||
}
|
||||
|
||||
fn peer_summary(
|
||||
discovered: bool,
|
||||
observed: bool,
|
||||
) -> gateway_admin::model::provider_credentials::ProviderQuota {
|
||||
gateway_admin::model::provider_credentials::ProviderQuota {
|
||||
upstream: Some(gateway_admin::model::upstream_info::ProviderUpstreamInfo {
|
||||
discovered,
|
||||
observed_at: observed.then(chrono::Utc::now),
|
||||
stale: true,
|
||||
connection_revision: "test-connection".to_owned(),
|
||||
last_error: None,
|
||||
pool: None,
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
use axum::{
|
||||
@@ -670,13 +687,17 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
for (authentication_kind, plan_type, plan_display) in [
|
||||
(Some("oauth"), Some("pro"), Some("Pro")),
|
||||
(Some("api_key"), None, None),
|
||||
(None, None, None),
|
||||
for (authentication_kind, plan_type, plan_display, peer) in [
|
||||
(Some("oauth"), Some("pro"), Some("Pro"), None),
|
||||
(Some("api_key"), None, None, None),
|
||||
(Some("api_key"), None, None, Some((true, true))),
|
||||
(Some("api_key"), None, None, Some((true, false))),
|
||||
(Some("api_key"), None, None, Some((false, true))),
|
||||
(None, None, None, None),
|
||||
] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
|
||||
*fixture.diagnostics.lock().expect("diagnostics") = DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 4,
|
||||
@@ -718,6 +739,10 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["accountUpstreamPeer"],
|
||||
json!(peer.map(|(d, o)| d && o))
|
||||
);
|
||||
assert_eq!(value["data"]["items"][0]["requestShare"], 0.5);
|
||||
assert_eq!(value["data"]["items"][0]["retryCount"], 3);
|
||||
assert_eq!(value["data"]["items"][0]["retryRate"], 0.5);
|
||||
@@ -906,6 +931,7 @@ async fn usage_route_projects_request_facts_and_billing() {
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
@@ -1131,6 +1157,39 @@ async fn usage_route_projects_request_facts_and_billing() {
|
||||
value["data"]["items"][2]["billing"]["totalAmountDisplay"],
|
||||
"$0.007"
|
||||
);
|
||||
|
||||
// 互联标识随当前摘要变化，重复请求共享同一账号，既有请求快照保持原值
|
||||
for peer in [
|
||||
Some((true, true)),
|
||||
Some((true, false)),
|
||||
Some((false, true)),
|
||||
None,
|
||||
] {
|
||||
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?status=all")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_peer_badge")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
for item in value["data"]["items"].as_array().unwrap() {
|
||||
assert_eq!(
|
||||
item["accountUpstreamPeer"],
|
||||
json!(peer.map(|(d, o)| d && o))
|
||||
);
|
||||
assert_eq!(item["accountName"], "Snapshot Alpha");
|
||||
assert_eq!(item["accountPlanType"], "supergrok_plus");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -57,6 +57,7 @@ fn update_body() -> Value {
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"maxRetryAfterSeconds": 240,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
|
||||
@@ -92,6 +92,7 @@ pub(super) async fn fixture() -> AdminTestFixture {
|
||||
|
||||
fn usage_record() -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
provider_account_upstream_peer: None,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
client_status_code: None,
|
||||
|
||||
@@ -261,6 +261,114 @@ async fn exports_scoped_observations_even_when_the_key_budget_is_exhausted_witho
|
||||
assert!(fixture.observations.lock().unwrap().summaries.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_plans_use_account_plan_fallback_independently_of_quota_sharing() {
|
||||
for share_quota in [true, false] {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
for (account_plan, quota_plan, expected) in [
|
||||
(None, Some("free"), Some("Free")),
|
||||
(Some("unknown"), Some("free"), Some("Free")),
|
||||
(Some(" "), Some("free"), Some("Free")),
|
||||
(Some("edu_pro"), Some("edu"), Some("EduPro")),
|
||||
(Some("free"), None, Some("Free")),
|
||||
(None, None, None),
|
||||
(None, Some("unknown"), None),
|
||||
] {
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = account_plan.map(str::to_owned);
|
||||
fixture
|
||||
.provider_quota
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.plan_type = quota_plan.map(str::to_owned);
|
||||
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
let plans =
|
||||
expected.map_or_else(|| json!([]), |name| json!([{ "name": name, "count": 1 }]));
|
||||
assert_eq!(
|
||||
body["pool"]["plans"], plans,
|
||||
"{account_plan:?} / {quota_plan:?}"
|
||||
);
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body.get("quota").is_some(), share_quota);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type
|
||||
.as_deref(),
|
||||
account_plan,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_only_summary_limits_exported_data_and_preserves_unknown_on_quota_read_failure() {
|
||||
let (fixture, app) = fixture(
|
||||
ClientPeerSharing {
|
||||
share_plans: true,
|
||||
share_quota: false,
|
||||
..sharing()
|
||||
},
|
||||
RAW_KEY,
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.account
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.account
|
||||
.plan_type = None;
|
||||
{
|
||||
let mut quota = fixture.provider_quota.lock().unwrap();
|
||||
let quota = quota.as_mut().unwrap();
|
||||
quota.plan_type = Some("free".into());
|
||||
quota.windows =
|
||||
vec![quota.windows[0].clone(); gateway_protocol::peer::MAX_QUOTA_WINDOWS + 1];
|
||||
}
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(
|
||||
body["pool"]["plans"],
|
||||
json!([{ "name": "Free", "count": 1 }])
|
||||
);
|
||||
assert_eq!(body["capabilities"], json!(["key_usage", "pool_summary"]));
|
||||
assert!(body.get("quota").is_none());
|
||||
|
||||
*fixture.provider_quota.lock().unwrap() = None;
|
||||
let response = query(&app, PATH, Some(RAW_KEY)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["pool"]["account_count"], 1);
|
||||
assert_eq!(body["pool"]["plans"], json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requires_authentication_explicit_sharing_and_bound_groups_and_rejects_query_overrides() {
|
||||
let (fixture, app) = fixture(sharing(), RAW_KEY).await;
|
||||
|
||||
@@ -431,6 +431,9 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
values("x-codex-installation-id"),
|
||||
vec![b"client-installation".to_vec()]
|
||||
);
|
||||
for name in ["x-openai-account-routing-override", "x-openai-fedramp"] {
|
||||
assert_eq!(values(name), vec![b"first".to_vec(), b"second".to_vec()]);
|
||||
}
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
@@ -450,8 +453,6 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-openai-account-routing-override",
|
||||
"x-openai-fedramp",
|
||||
// 上游指纹由运行时画像统一生成，客户端不得覆盖
|
||||
"user-agent",
|
||||
"originator",
|
||||
|
||||
@@ -1677,6 +1677,10 @@ where
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
// 服务器建议不可截短；超出请求冻结的上限后沿用安全换号或原错误出口
|
||||
let retry_after_exceeds_limit = error
|
||||
.retry_after()
|
||||
.is_some_and(|delay| delay > self.plan.max_retry_after());
|
||||
// Provider 的有界同账号传输恢复是尽力重试策略，不要求把 Sent/Ambiguous
|
||||
// 改写成安全证明；外部副作用、取消、预算与下游提交仍由 Core 拦截
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
@@ -1699,7 +1703,8 @@ where
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if !execution_effect_observed
|
||||
}) if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1707,7 +1712,8 @@ where
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if !execution_effect_observed
|
||||
if !retry_after_exceeds_limit
|
||||
&& !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
@@ -1753,7 +1759,7 @@ where
|
||||
max_delay,
|
||||
jitter,
|
||||
..
|
||||
}) if ordinary_retry && !suppress_transient_retry => {
|
||||
}) if ordinary_retry && !suppress_transient_retry && !retry_after_exceeds_limit => {
|
||||
let retries = self
|
||||
.transient_retry_counts
|
||||
.entry(current.metadata.provider_account_id().clone())
|
||||
@@ -1832,6 +1838,8 @@ where
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"retryAfterExceeded": retry_after_exceeds_limit,
|
||||
"maxRetryAfterSeconds": self.plan.max_retry_after().as_secs(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
"connectionRetry": connection_retry_requested,
|
||||
|
||||
@@ -576,6 +576,7 @@ pub struct RoutingPlan {
|
||||
account_error_protection: Arc<crate::account::CompiledAccountErrorProtection>,
|
||||
account_error_protection_revision: u64,
|
||||
request_body_limit: usize,
|
||||
max_retry_after: std::time::Duration,
|
||||
pricing: Arc<crate::metering::PricingOverrides>,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
privacy: Option<Arc<dyn crate::settings::privacy::CompiledPrivacyPolicy>>,
|
||||
@@ -598,6 +599,12 @@ impl RoutingPlan {
|
||||
Arc::clone(&self.account_error_protection)
|
||||
}
|
||||
|
||||
/// 本次请求接受的单次 Retry-After 等待上限
|
||||
#[must_use]
|
||||
pub const fn max_retry_after(&self) -> std::time::Duration {
|
||||
self.max_retry_after
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> usize {
|
||||
self.request_body_limit
|
||||
|
||||
@@ -23,7 +23,7 @@ use super::{
|
||||
RuntimeAccountDirectory, UpstreamModelId,
|
||||
};
|
||||
|
||||
const MAXIMUM_CATALOG_STABILITY_ATTEMPTS: usize = 4;
|
||||
const MAXIMUM_SNAPSHOT_STABILITY_ATTEMPTS: usize = 4;
|
||||
|
||||
type ModelCatalogAccounts = BTreeMap<ProviderKind, BTreeMap<String, BTreeSet<ProviderAccountId>>>;
|
||||
|
||||
@@ -328,14 +328,17 @@ impl RuntimeSnapshotCompiler {
|
||||
ConfigRevision,
|
||||
crate::routing::extensions::ExtensionSetReference,
|
||||
)> = None;
|
||||
for _ in 0..MAXIMUM_CATALOG_STABILITY_ATTEMPTS {
|
||||
let mut instability = RuntimeSnapshotCompileError::CatalogChanged;
|
||||
for _ in 0..MAXIMUM_SNAPSHOT_STABILITY_ATTEMPTS {
|
||||
let facts = self
|
||||
.store
|
||||
.load_snapshot_facts()
|
||||
.await
|
||||
.map_err(|_| RuntimeSnapshotCompileError::StoreUnavailable)?;
|
||||
if facts.config_revision != facts.observed_current_revision {
|
||||
return Err(RuntimeSnapshotCompileError::RevisionChanged);
|
||||
// 并发提交使本轮事实过期时重读完整快照，不能混用不同版本的权限
|
||||
instability = RuntimeSnapshotCompileError::RevisionChanged;
|
||||
continue;
|
||||
}
|
||||
let revision = facts.config_revision;
|
||||
let extensions = match &self.extensions {
|
||||
@@ -347,7 +350,7 @@ impl RuntimeSnapshotCompiler {
|
||||
*prepared_revision != revision || !prepared.is_ready()
|
||||
});
|
||||
if must_prepare {
|
||||
let prepared = preparer
|
||||
let prepared = match preparer
|
||||
.prepare_with_request_body_limit(
|
||||
revision,
|
||||
usize::try_from(
|
||||
@@ -356,7 +359,21 @@ impl RuntimeSnapshotCompiler {
|
||||
.unwrap_or(usize::MAX),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| RuntimeSnapshotCompileError::ExtensionsUnavailable)?;
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(_) => {
|
||||
// 扩展读取也可能遇到并发提交；只有版本已变才重读事实
|
||||
let observed =
|
||||
self.store.current_config_revision().await.map_err(|_| {
|
||||
RuntimeSnapshotCompileError::StoreUnavailable
|
||||
})?;
|
||||
if observed == revision {
|
||||
return Err(RuntimeSnapshotCompileError::ExtensionsUnavailable);
|
||||
}
|
||||
instability = RuntimeSnapshotCompileError::RevisionChanged;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
prepared_extensions = Some((revision, prepared));
|
||||
}
|
||||
prepared_extensions
|
||||
@@ -390,7 +407,8 @@ impl RuntimeSnapshotCompiler {
|
||||
.map_err(|_| RuntimeSnapshotCompileError::StoreUnavailable)?
|
||||
!= revision
|
||||
{
|
||||
return Err(RuntimeSnapshotCompileError::RevisionChanged);
|
||||
instability = RuntimeSnapshotCompileError::RevisionChanged;
|
||||
continue;
|
||||
}
|
||||
let snapshot = snapshot
|
||||
.with_provider_catalog_generations(if cached.is_some() && !refresh_catalog {
|
||||
@@ -401,8 +419,9 @@ impl RuntimeSnapshotCompiler {
|
||||
.with_extensions(extensions);
|
||||
return Ok(snapshot);
|
||||
}
|
||||
instability = RuntimeSnapshotCompileError::CatalogChanged;
|
||||
}
|
||||
Err(RuntimeSnapshotCompileError::CatalogChanged)
|
||||
Err(instability)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1408,6 +1427,9 @@ impl RuntimeSnapshot {
|
||||
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
@@ -1495,6 +1517,9 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
max_retry_after: std::time::Duration::from_secs(u64::from(
|
||||
self.settings.values.max_retry_after_seconds,
|
||||
)),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
|
||||
@@ -21,6 +21,7 @@ pub struct SettingsValues {
|
||||
pub(crate) max_waiting_per_key: u32,
|
||||
pub(crate) max_waiting_per_account: u32,
|
||||
pub(crate) concurrency_wait_timeout_seconds: u32,
|
||||
pub(crate) max_retry_after_seconds: u32,
|
||||
pub(crate) openai_guardian_reserved_concurrency: u32,
|
||||
pub(crate) openai_account_affinity: crate::account::AccountAffinity,
|
||||
pub(crate) max_account_rotations: u32,
|
||||
@@ -84,6 +85,13 @@ impl SettingsValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// 网关接受的单次服务器建议等待上限，超过时交由既有换号或错误出口处理
|
||||
#[must_use]
|
||||
pub const fn with_max_retry_after_seconds(mut self, seconds: u32) -> Self {
|
||||
self.max_retry_after_seconds = seconds;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
|
||||
self.max_account_rotations = rotations;
|
||||
@@ -178,6 +186,7 @@ impl SettingsValues {
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: crate::account::AccountAffinity::default(),
|
||||
max_account_rotations: 3,
|
||||
|
||||
@@ -1174,6 +1174,7 @@ fn plan_with_profiles(
|
||||
fast_mode,
|
||||
profiles,
|
||||
Default::default(),
|
||||
240,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1184,6 +1185,7 @@ fn plan_with_protection(
|
||||
fast_mode: FastMode,
|
||||
profiles: BTreeMap<ProviderKind, gateway_core::account::OpaqueProviderData>,
|
||||
protection: gateway_core::account::AccountErrorProtection,
|
||||
max_retry_after_seconds: u32,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
@@ -1237,7 +1239,8 @@ fn plan_with_protection(
|
||||
.unwrap(),
|
||||
)
|
||||
.with_request_location(request_location, true)
|
||||
.with_account_error_protection(protection),
|
||||
.with_account_error_protection(protection)
|
||||
.with_max_retry_after_seconds(max_retry_after_seconds),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -4216,6 +4219,177 @@ fn server_retry_advice_should_override_local_backoff_and_transport_fallback() {
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_after_plan(operation: &Operation, limit: u32) -> RoutingPlan {
|
||||
plan_with_protection(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::MIN),
|
||||
Default::default(),
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
limit,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_at_or_below_the_configured_limit_keeps_the_full_wait() {
|
||||
for (limit, seconds) in [(240, 240), (60, 60), (400, 308)] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transient_rejection().with_retry_after(Duration::from_secs(seconds))
|
||||
)],
|
||||
}]);
|
||||
let cancellation = CancellationToken::new();
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(600)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
cancellation.clone(),
|
||||
))
|
||||
.unwrap();
|
||||
let trace = session.trace().clone();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(block_on(async { futures::poll!(&mut collection) }).is_pending());
|
||||
let trace = trace.snapshot().unwrap();
|
||||
let retry = trace["events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "retry.decided")
|
||||
.unwrap();
|
||||
assert_eq!(retry["data"]["delayMs"], seconds * 1000);
|
||||
assert_eq!(retry["data"]["retryAfterExceeded"], false);
|
||||
cancellation.cancel();
|
||||
assert!(matches!(block_on(collection), Err(EngineError::Cancelled)));
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_above_the_configured_limit_rotates_without_waiting() {
|
||||
for (limit, seconds) in [(240, 241), (240, 308), (60, 61), (0, 1)] {
|
||||
for rejection in [
|
||||
transient_rejection(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_fallback(),
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(seconds)),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = retry_after_plan(&operation, limit);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection.with_retry_after(Duration::from_secs(seconds)))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
assert!(matches!(
|
||||
block_on(async { futures::poll!(&mut collection) }),
|
||||
std::task::Poll::Ready(Ok(_))
|
||||
));
|
||||
drop(collection);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(contexts[1].required_account(), None);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excessive_retry_after_preserves_the_error_when_rotation_is_unavailable_or_disallowed() {
|
||||
for (rejection, rotation_allowed) in [
|
||||
(transient_rejection(), true),
|
||||
(
|
||||
transient_rejection().with_transient_retry(
|
||||
NonZeroU32::MIN,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
),
|
||||
false,
|
||||
),
|
||||
(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_transport_retry(NonZeroU32::MIN, Duration::from_secs(308)),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let original = Bytes::from_static(
|
||||
br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"308"}}}"#,
|
||||
);
|
||||
let expected_kind = rejection.kind();
|
||||
let rejection = rejection
|
||||
.with_retry_after(Duration::from_secs(308))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/json".to_vec()),
|
||||
original.clone(),
|
||||
));
|
||||
let mut scripts = vec![Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(rejection)],
|
||||
}];
|
||||
if rotation_allowed {
|
||||
scripts.push(Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)));
|
||||
}
|
||||
let (coordinator, _, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
let result = block_on(async { futures::poll!(&mut collection) });
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) = result else {
|
||||
panic!("must return the original failure without waiting")
|
||||
};
|
||||
assert_eq!(error.kind(), expected_kind);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(308)));
|
||||
assert_eq!(
|
||||
error.client_visible_upstream_response().unwrap().body(),
|
||||
&original
|
||||
);
|
||||
assert_eq!(
|
||||
provider.contexts.lock().unwrap().len(),
|
||||
if rotation_allowed { 2 } else { 1 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
for policy in [
|
||||
@@ -4223,7 +4397,7 @@ fn zero_server_retry_advice_should_override_nonzero_local_backoff() {
|
||||
TransientRetryPolicy::RetryThenReturnError,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let route_plan = retry_after_plan(&operation, 0);
|
||||
let (coordinator, _, _) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
@@ -4944,7 +5118,9 @@ fn structural_event_before_replay_safe_failure_should_switch_account_before_comm
|
||||
|
||||
#[test]
|
||||
fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery() {
|
||||
for close_window in [false, true] {
|
||||
for (close_window, excessive_retry_after) in
|
||||
[(false, false), (true, false), (false, true), (true, true)]
|
||||
{
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let prelude = || {
|
||||
@@ -4971,11 +5147,12 @@ fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery(
|
||||
// 后续前导不能撤销已经交付未知事件的事实
|
||||
items.push(Ok(prelude()));
|
||||
}
|
||||
items.push(Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe()));
|
||||
items.push(Err(if excessive_retry_after {
|
||||
transient_rejection().with_retry_after(Duration::from_secs(308))
|
||||
} else {
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
|
||||
.with_replay_safe()
|
||||
}));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
@@ -7185,6 +7362,7 @@ fn error_protection_tracks_real_retry_attempts_and_skips_diagnostic_runs() {
|
||||
FastMode::Default,
|
||||
BTreeMap::new(),
|
||||
policy().as_ref().clone(),
|
||||
240,
|
||||
);
|
||||
let feedback = Arc::new(Feedback::default());
|
||||
let failure = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
|
||||
@@ -730,12 +730,116 @@ impl ProviderCatalogPort for PublishingCatalog {
|
||||
|
||||
#[test]
|
||||
fn compiler_should_reject_revision_changed_during_consistent_read() {
|
||||
let facts = facts(1, 2);
|
||||
let compiler = compiler(Arc::new(TestSnapshotStore::new(Ok(facts))));
|
||||
let store = Arc::new(SequencedSnapshotStore {
|
||||
facts: vec![facts(1, 2)],
|
||||
loads: AtomicUsize::new(0),
|
||||
current_revision: revision(2),
|
||||
});
|
||||
let compiler = compiler(store.clone());
|
||||
|
||||
let error = block_on(compiler.compile()).expect_err("revision drift must fail closed");
|
||||
|
||||
assert_eq!(error, RuntimeSnapshotCompileError::RevisionChanged);
|
||||
assert_eq!(store.loads.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compiler_reloads_complete_facts_after_revision_changes_during_read() {
|
||||
let store = Arc::new(SequencedSnapshotStore {
|
||||
facts: vec![facts(1, 2), facts(2, 2)],
|
||||
loads: AtomicUsize::new(0),
|
||||
current_revision: revision(2),
|
||||
});
|
||||
|
||||
let snapshot = block_on(compiler(store.clone()).compile()).expect("latest consistent facts");
|
||||
|
||||
assert_eq!(snapshot.revision(), revision(2));
|
||||
assert_eq!(store.loads.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compiler_reloads_revision_changed_after_extension_preparation() {
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
let extensions = Arc::new(CountingExtensionPreparation {
|
||||
prepares: AtomicUsize::new(0),
|
||||
drops: Arc::clone(&drops),
|
||||
});
|
||||
let compiler = RuntimeSnapshotCompiler::new(
|
||||
Arc::new(SequencedSnapshotStore {
|
||||
facts: vec![facts(1, 1), facts(2, 2)],
|
||||
loads: AtomicUsize::new(0),
|
||||
current_revision: revision(2),
|
||||
}),
|
||||
Arc::new(TestCatalog::Empty),
|
||||
)
|
||||
.with_extensions(extensions.clone());
|
||||
|
||||
let snapshot = block_on(compiler.compile()).expect("latest prepared snapshot");
|
||||
|
||||
assert_eq!(snapshot.revision(), revision(2));
|
||||
assert_eq!(snapshot.extensions().unwrap().id().as_str(), "prepared-2");
|
||||
assert_eq!(extensions.prepares.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1);
|
||||
drop(snapshot);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compiler_rechecks_revision_when_extension_preparation_rejects_stale_facts() {
|
||||
struct RacingPreparation {
|
||||
store: Arc<TestSnapshotStore>,
|
||||
changes: usize,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl ExtensionPreparationPort for RacingPreparation {
|
||||
fn prepare(
|
||||
&self,
|
||||
_: ConfigRevision,
|
||||
) -> BoxFuture<'_, Result<ExtensionSetReference, ExtensionPreparationError>> {
|
||||
Box::pin(async move {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.changes == 0 || call < self.changes {
|
||||
if self.changes != 0 {
|
||||
let next = u64::try_from(call).unwrap() + 2;
|
||||
*self.store.facts.lock().unwrap() = Ok(facts(next, next));
|
||||
*self.store.current_revision.lock().unwrap() = Ok(revision(next));
|
||||
}
|
||||
return Err(ExtensionPreparationError);
|
||||
}
|
||||
Ok(ExtensionSetReference::new(
|
||||
ExtensionSetId::new("ready".to_owned()).unwrap(),
|
||||
Arc::new(AliasLease(vec![])),
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (changes, expected_calls) in [(0, 1), (1, 2), (4, 4)] {
|
||||
let store = Arc::new(TestSnapshotStore::new(Ok(facts(1, 1))));
|
||||
let extensions = Arc::new(RacingPreparation {
|
||||
store: store.clone(),
|
||||
changes,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let compiler = RuntimeSnapshotCompiler::new(store, Arc::new(TestCatalog::Empty))
|
||||
.with_extensions(extensions.clone());
|
||||
|
||||
let result = block_on(compiler.compile());
|
||||
|
||||
match changes {
|
||||
0 => assert_eq!(
|
||||
result.unwrap_err(),
|
||||
RuntimeSnapshotCompileError::ExtensionsUnavailable
|
||||
),
|
||||
1 => assert_eq!(result.unwrap().revision(), revision(2)),
|
||||
_ => assert_eq!(
|
||||
result.unwrap_err(),
|
||||
RuntimeSnapshotCompileError::RevisionChanged
|
||||
),
|
||||
}
|
||||
assert_eq!(extensions.calls.load(Ordering::SeqCst), expected_calls);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -326,6 +326,39 @@ fn publisher_should_suspend_but_still_notify_after_committed_refresh_failure() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_refresh_reloads_raced_facts_without_losing_revoked_scope() {
|
||||
block_on(async {
|
||||
let store = Arc::new(TestSnapshotStore::new(Ok(scoped_facts(10, true))));
|
||||
let initial = compiler(store.clone())
|
||||
.compile()
|
||||
.await
|
||||
.expect("initial scope");
|
||||
let handle = RuntimeSnapshotHandle::new(initial);
|
||||
let frozen = handle.acquire().expect("initial request");
|
||||
assert!(removed_account_allowed(&frozen));
|
||||
let publisher = RuntimeSnapshotPublisher::new(
|
||||
Arc::new(compiler(store.clone())),
|
||||
handle.clone(),
|
||||
Arc::new(TestSnapshotSubscriptions::default()),
|
||||
);
|
||||
let (release, gate) = oneshot::channel();
|
||||
*store.next_load.lock().expect("load gate lock") = Some(gate);
|
||||
let mut committed = publisher.publish_committed(revision(11));
|
||||
assert!(futures::poll!(committed.as_mut()).is_pending());
|
||||
*store.facts.lock().expect("facts lock") = Ok(scoped_facts(11, false));
|
||||
*store.current_revision.lock().expect("revision lock") = Ok(revision(11));
|
||||
release.send(Ok(facts(10, 11))).expect("release raced read");
|
||||
|
||||
committed.await;
|
||||
|
||||
let current = handle.acquire().expect("latest snapshot remains available");
|
||||
assert_eq!(current.revision(), revision(11));
|
||||
assert!(!removed_account_allowed(¤t));
|
||||
assert!(removed_account_allowed(&frozen));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publisher_should_contribute_reconciliation_and_subscription_workers() {
|
||||
let store = Arc::new(TestSnapshotStore::new(Ok(facts(1, 1))));
|
||||
|
||||
@@ -125,6 +125,17 @@ impl HttpProxyProbe {
|
||||
}
|
||||
|
||||
impl HttpProxyProbe {
|
||||
async fn timed_exit_ip_at(
|
||||
&self,
|
||||
proxy: Option<&OutboundProxy>,
|
||||
endpoint: &str,
|
||||
) -> (Result<IpAddr, &'static str>, u64) {
|
||||
let started = Instant::now();
|
||||
let result = self.exit_ip_at(proxy, endpoint).await;
|
||||
let latency_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
(result, latency_ms)
|
||||
}
|
||||
|
||||
async fn test_connection(&self, proxy: Option<&OutboundProxy>) -> ProxyTestResult {
|
||||
let started = Instant::now();
|
||||
let timeout_limit = Duration::from_secs(15);
|
||||
@@ -169,15 +180,15 @@ impl HttpProxyProbe {
|
||||
} => {
|
||||
let probe_dual = async {
|
||||
tokio::join!(
|
||||
self.exit_ip_at(proxy, ipv4_endpoint),
|
||||
self.exit_ip_at(proxy, ipv6_endpoint),
|
||||
self.timed_exit_ip_at(proxy, ipv4_endpoint),
|
||||
self.timed_exit_ip_at(proxy, ipv6_endpoint),
|
||||
)
|
||||
};
|
||||
let result = tokio::time::timeout(timeout_limit, probe_dual).await;
|
||||
let latency_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
match result {
|
||||
Ok((res_v4, res_v6)) => {
|
||||
Ok(((res_v4, latency_v4), (res_v6, latency_v6))) => {
|
||||
let exit_ipv4: Option<Ipv4Addr> = match res_v4 {
|
||||
Ok(IpAddr::V4(v4)) => Some(v4),
|
||||
_ => None,
|
||||
@@ -186,6 +197,14 @@ impl HttpProxyProbe {
|
||||
Ok(IpAddr::V6(v6)) => Some(v6),
|
||||
_ => None,
|
||||
};
|
||||
// 延迟对应首选的可用出口，另一地址族失败等待不代表该出口延迟
|
||||
let latency_ms = if exit_ipv4.is_some() {
|
||||
latency_v4
|
||||
} else if exit_ipv6.is_some() {
|
||||
latency_v6
|
||||
} else {
|
||||
latency_ms
|
||||
};
|
||||
|
||||
if exit_ipv4.is_some() && exit_ipv6.is_some() {
|
||||
ProxyTestResult {
|
||||
|
||||
@@ -6,7 +6,11 @@ use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use super::{OperationError, internal, invalid};
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const APP_BINARY_NAME: &str = if cfg!(windows) {
|
||||
"codex-proxy-rs.exe"
|
||||
} else {
|
||||
"codex-proxy-rs"
|
||||
};
|
||||
const MAX_EXTRACTED_SIZE: u64 = 1024 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_FILES: usize = 20_000;
|
||||
|
||||
@@ -126,14 +130,42 @@ pub(crate) fn extract_release(
|
||||
|
||||
fn unsafe_archive_path(path: &Path) -> bool {
|
||||
path.is_absolute()
|
||||
|| path.components().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::ParentDir | Component::Prefix(_) | Component::RootDir
|
||||
)
|
||||
|| path.components().any(|component| match component {
|
||||
Component::ParentDir | Component::Prefix(_) | Component::RootDir => true,
|
||||
#[cfg(windows)]
|
||||
Component::Normal(name) => unsafe_windows_name(name),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn unsafe_windows_name(name: &std::ffi::OsStr) -> bool {
|
||||
let name = name.to_string_lossy();
|
||||
// 防止归档路径被 Win32 解释为设备、NTFS 备用数据流或另一个规范化文件名
|
||||
if name.ends_with(['.', ' '])
|
||||
|| name
|
||||
.chars()
|
||||
.any(|value| value < ' ' || "<>:\"|?*".contains(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let stem = name
|
||||
.split('.')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_uppercase();
|
||||
matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
|
||||
|| stem
|
||||
.strip_prefix("COM")
|
||||
.or_else(|| stem.strip_prefix("LPT"))
|
||||
.is_some_and(|suffix| {
|
||||
matches!(
|
||||
suffix,
|
||||
"1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn web_dist_relative_path(path: &Path) -> Option<PathBuf> {
|
||||
let components = path
|
||||
.components()
|
||||
|
||||
@@ -16,6 +16,7 @@ mod swap_paths;
|
||||
|
||||
pub use config::SystemUpdateConfig;
|
||||
pub use operations::ProcessSystemOperations;
|
||||
pub use process::startup_arguments;
|
||||
pub use release::validate_download_url;
|
||||
|
||||
use error::{OperationError, conflict, internal, invalid, upstream};
|
||||
|
||||
@@ -41,12 +41,18 @@ pub struct ProcessSystemOperations {
|
||||
cancellation: CancellationToken,
|
||||
events: Arc<UpdateEvents>,
|
||||
config: Arc<SystemUpdateConfig>,
|
||||
operation_lock: Arc<AsyncMutex<Option<OperationFileLock>>>,
|
||||
operation_lock: Arc<AsyncMutex<Option<RestartGuard>>>,
|
||||
release_cache: Arc<ReleaseCache>,
|
||||
running_files: Arc<Result<ReleaseFiles, OperationError>>,
|
||||
diagnostics: super::diagnostics::Diagnostics,
|
||||
}
|
||||
|
||||
struct RestartGuard {
|
||||
// 字段按声明顺序释放，先移除更新锁，再通知替换进程开始启动
|
||||
_file_lock: OperationFileLock,
|
||||
_replacement_input: Option<std::process::ChildStdin>,
|
||||
}
|
||||
|
||||
impl ProcessSystemOperations {
|
||||
#[must_use]
|
||||
pub fn new(cancellation: CancellationToken, mut config: SystemUpdateConfig) -> Self {
|
||||
@@ -75,7 +81,7 @@ impl ProcessSystemOperations {
|
||||
self.diagnostics.bind(diagnostics);
|
||||
}
|
||||
|
||||
fn lock_operation(&self) -> Result<OwnedMutexGuard<Option<OperationFileLock>>, OperationError> {
|
||||
fn lock_operation(&self) -> Result<OwnedMutexGuard<Option<RestartGuard>>, OperationError> {
|
||||
let operation = Arc::clone(&self.operation_lock)
|
||||
.try_lock_owned()
|
||||
.map_err(|_| conflict("已有系统操作正在执行"))?;
|
||||
@@ -559,15 +565,17 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
{
|
||||
return Err(conflict("重启检查期间安装文件已变化，请重新确认"));
|
||||
}
|
||||
let message = if self.config.deployment_mode == "docker" {
|
||||
"已安排进程内重启"
|
||||
let (message, replacement_input) = if self.config.deployment_mode == "docker" {
|
||||
("已安排进程内重启", None)
|
||||
} else {
|
||||
spawn_replacement(&self.config)?;
|
||||
"已安排自重启"
|
||||
("已安排自重启", spawn_replacement(&self.config)?)
|
||||
};
|
||||
let operation_id = operation_id("restart");
|
||||
// 受理后直到旧服务释放，保留重启状态和跨进程锁；延迟关闭只用于送达响应
|
||||
*operation = Some(file_lock);
|
||||
*operation = Some(RestartGuard {
|
||||
_file_lock: file_lock,
|
||||
_replacement_input: replacement_input,
|
||||
});
|
||||
let cancellation = self.cancellation.clone();
|
||||
drop(tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
@@ -2,15 +2,37 @@
|
||||
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::io;
|
||||
use std::process::{ChildStdin, Command, Stdio};
|
||||
|
||||
#[cfg(unix)]
|
||||
use super::config::environment_value;
|
||||
use super::{OperationError, SystemUpdateConfig, internal};
|
||||
|
||||
#[cfg(unix)]
|
||||
const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS";
|
||||
#[cfg(unix)]
|
||||
const DEFAULT_RESTART_DELAY_MS: u64 = 1_200;
|
||||
const RESTART_HANDOFF_ARGUMENT: &str = "--restart-handoff";
|
||||
|
||||
pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), OperationError> {
|
||||
/// 组合根在创建运行时前接收重启交接，并取得不含内部交接标记的 CLI 参数
|
||||
pub fn startup_arguments() -> Result<impl Iterator<Item = std::ffi::OsString>, OperationError> {
|
||||
let mut arguments = env::args_os().skip(1).peekable();
|
||||
if arguments
|
||||
.peek()
|
||||
.is_some_and(|value| value == RESTART_HANDOFF_ARGUMENT)
|
||||
{
|
||||
arguments.next();
|
||||
// 旧服务释放操作锁后关闭管道；旧进程异常退出也会关闭管道，避免固定延迟抢占端口
|
||||
io::copy(&mut io::stdin().lock(), &mut io::sink())
|
||||
.map_err(|error| internal("无法等待旧服务释放资源").with_source(error))?;
|
||||
}
|
||||
Ok(arguments)
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_replacement(
|
||||
config: &SystemUpdateConfig,
|
||||
) -> Result<Option<ChildStdin>, OperationError> {
|
||||
let executable = config.executable_path()?;
|
||||
let metadata = fs::metadata(&executable)
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))?;
|
||||
@@ -25,12 +47,12 @@ pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), Opera
|
||||
}
|
||||
}
|
||||
|
||||
let delay_ms = environment_value(RESTART_DELAY_ENV)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_RESTART_DELAY_MS);
|
||||
#[cfg(unix)]
|
||||
let mut command = {
|
||||
let delay_ms = environment_value(RESTART_DELAY_ENV)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_RESTART_DELAY_MS);
|
||||
let delay = format!("{}.{:03}", delay_ms / 1_000, delay_ms % 1_000);
|
||||
let mut command = Command::new("/bin/sh");
|
||||
command
|
||||
@@ -39,20 +61,26 @@ pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), Opera
|
||||
.arg("codex-proxy-rs-restart")
|
||||
.arg(delay)
|
||||
.arg(&executable)
|
||||
.args(env::args_os().skip(1));
|
||||
.args(env::args_os().skip(1))
|
||||
.stdin(Stdio::null());
|
||||
command
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let mut command = {
|
||||
let mut command = Command::new(&executable);
|
||||
command
|
||||
.args(env::args_os().skip(1))
|
||||
.env(RESTART_DELAY_ENV, delay_ms.to_string());
|
||||
.arg(RESTART_HANDOFF_ARGUMENT)
|
||||
// 连续重启时移除当前进程已消费的交接标记，保留原始参数及编码
|
||||
.args(
|
||||
env::args_os()
|
||||
.skip(1)
|
||||
.skip_while(|value| value == RESTART_HANDOFF_ARGUMENT),
|
||||
)
|
||||
.stdin(Stdio::piped());
|
||||
command
|
||||
};
|
||||
command.stdin(Stdio::null());
|
||||
command
|
||||
.spawn()
|
||||
.map(|_| ())
|
||||
.map(|mut child| child.stdin.take())
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl UpdateTempDir {
|
||||
|
||||
impl Drop for UpdateTempDir {
|
||||
fn drop(&mut self) {
|
||||
if let Err(error) = fs::remove_dir_all(&self.path)
|
||||
if let Err(error) = super::swap::remove_dir_all(&self.path)
|
||||
&& error.kind() != io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!(path = %self.path.display(), error = %error, "清理系统更新临时目录失败");
|
||||
|
||||
@@ -294,12 +294,40 @@ fn protect_official_plugins(_path: &Path) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_dir_all(path: &Path) -> io::Result<()> {
|
||||
pub(crate) fn remove_dir_all(path: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
make_directories_writable(path)?;
|
||||
#[cfg(windows)]
|
||||
make_files_writable(path)?;
|
||||
fs::remove_dir_all(path)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn make_files_writable(path: &Path) -> io::Result<()> {
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
if metadata.file_type().is_symlink() || (!metadata.is_dir() && !metadata.is_file()) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"release directory contains an unsupported file type",
|
||||
));
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
for entry in fs::read_dir(path)? {
|
||||
make_files_writable(&entry?.path())?;
|
||||
}
|
||||
} else if metadata.permissions().readonly() {
|
||||
// tar 的 0444 会成为 Windows 只读属性，清理受管备份前必须移除此属性
|
||||
let mut permissions = metadata.permissions();
|
||||
#[expect(
|
||||
clippy::permissions_set_readonly_false,
|
||||
reason = "仅 Windows 清理只读属性，不会放宽 Unix 权限"
|
||||
)]
|
||||
permissions.set_readonly(false);
|
||||
fs::set_permissions(path, permissions)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn make_directories_writable(path: &Path) -> io::Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
@@ -155,6 +155,39 @@ async fn dual_stack_proxy_probe_reports_single_stack_when_only_one_succeeds() {
|
||||
assert!(result.message.contains("仅 IPv4"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dual_stack_latency_measures_the_selected_exit_without_the_other_family_wait() {
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
for working_family in ["v4", "v6"] {
|
||||
let server = MockServer::start().await;
|
||||
for (family, ip) in [("v4", "203.0.113.8"), ("v6", "2001:db8::8")] {
|
||||
let response = if family == working_family {
|
||||
ResponseTemplate::new(200).set_body_json(json!({"ip": ip}))
|
||||
} else {
|
||||
ResponseTemplate::new(503).set_delay(Duration::from_millis(500))
|
||||
};
|
||||
Mock::given(path(format!("/{family}")))
|
||||
.respond_with(response)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
}
|
||||
let started = Instant::now();
|
||||
let result = HttpProxyProbe::new_dual(
|
||||
format!("{}/v4", server.uri()),
|
||||
format!("{}/v6", server.uri()),
|
||||
)
|
||||
.test(None, false)
|
||||
.await;
|
||||
assert!(result.success);
|
||||
assert_eq!(result.exit_ipv4.is_some(), working_family == "v4");
|
||||
assert_eq!(result.exit_ipv6.is_some(), working_family == "v6");
|
||||
assert!(started.elapsed() >= Duration::from_millis(500));
|
||||
assert!(u128::from(result.latency_ms) + 400 < started.elapsed().as_millis());
|
||||
}
|
||||
}
|
||||
|
||||
fn location_response(ip: &str, timezone: &str) -> serde_json::Value {
|
||||
json!({"success": true, "ip": ip, "country_code": "JP", "region": "Tokyo", "city": "Tokyo", "timezone": {"id": timezone}})
|
||||
}
|
||||
|
||||
@@ -6,13 +6,25 @@ use std::{
|
||||
};
|
||||
|
||||
fn main() {
|
||||
let executable = std::env::current_exe().unwrap();
|
||||
let directory = executable.parent().unwrap();
|
||||
if directory.join("restart-probe").is_file() {
|
||||
std::fs::write(directory.join("replacement-waiting"), b"ready").unwrap();
|
||||
let arguments = gateway_host::system_update::startup_arguments()
|
||||
.unwrap()
|
||||
.collect::<Vec<_>>();
|
||||
assert!(!arguments.iter().any(|value| value == "--restart-handoff"));
|
||||
assert!(!directory.join("update.lock").exists());
|
||||
std::fs::write(directory.join("replacement-ran"), b"started").unwrap();
|
||||
return;
|
||||
}
|
||||
let mode = std::fs::read_to_string("mode").unwrap();
|
||||
if mode == "stderr" {
|
||||
std::io::stderr().write_all(&[b'x'; 4096]).unwrap();
|
||||
}
|
||||
if matches!(
|
||||
mode.as_str(),
|
||||
"busy" | "stdin" | "stdout" | "exit" | "allocate"
|
||||
"busy" | "stdin" | "stdout" | "exit" | "allocate" | "update-probe"
|
||||
) {
|
||||
std::io::stdout().write_all(b"R").unwrap();
|
||||
std::io::stdout().flush().unwrap();
|
||||
@@ -25,6 +37,11 @@ fn main() {
|
||||
std::io::stdin().read_exact(&mut [0; 1]).unwrap();
|
||||
std::fs::write("unblocked", b"stdin").unwrap();
|
||||
}
|
||||
"update-probe" => {
|
||||
std::io::stdin().read_exact(&mut [0; 1]).unwrap();
|
||||
std::fs::write("unblocked", b"still-running").unwrap();
|
||||
return;
|
||||
}
|
||||
"stdout" => {
|
||||
// 测试端不消费此正文，超过管道容量后必须由监督器终止，而不是等待写入完成
|
||||
std::io::stdout()
|
||||
|
||||
@@ -18,8 +18,7 @@ async fn manual_image_replacement_should_clear_pending_state_and_invalid_backup(
|
||||
assert!(complete_update(&service, TARGET_VERSION).await.need_restart);
|
||||
// 模拟重建镜像：数据卷保留，应用文件和容器内备份被重新替换
|
||||
fixture.write_executable("manually-installed-binary");
|
||||
fs::remove_file(fixture.root.path().join("codex-proxy-rs.backup"))
|
||||
.expect("remove image backup");
|
||||
fs::remove_file(fixture.backup()).expect("remove image backup");
|
||||
let mut config = fixture.config(&format!("{}/repos", server.uri()));
|
||||
config.version = "1.1.0".into();
|
||||
config.deployment_mode = "docker".into();
|
||||
@@ -126,11 +125,7 @@ async fn pending_update_should_revoke_rollback_when_backup_content_changes() {
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
assert!(complete_update(&service, TARGET_VERSION).await.need_restart);
|
||||
fs::write(
|
||||
fixture.root.path().join("codex-proxy-rs.backup"),
|
||||
"unrelated-backup",
|
||||
)
|
||||
.expect("changed backup");
|
||||
fs::write(fixture.backup(), "unrelated-backup").expect("changed backup");
|
||||
let status = service
|
||||
.update_status()
|
||||
.await
|
||||
|
||||
@@ -40,6 +40,12 @@ use tar::{Builder, EntryType, Header};
|
||||
use wiremock::matchers::{method, path, query_param};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
const BINARY_NAME: &str = if cfg!(windows) {
|
||||
"codex-proxy-rs.exe"
|
||||
} else {
|
||||
"codex-proxy-rs"
|
||||
};
|
||||
|
||||
const OLD_RELEASE_MANIFEST: &str = r#"{"gateway_version":"1.0.0"}"#;
|
||||
|
||||
const TARGET_VERSION: &str = "1.9.9";
|
||||
@@ -253,7 +259,11 @@ impl Fixture {
|
||||
}
|
||||
|
||||
fn executable(&self) -> PathBuf {
|
||||
self.root.path().join("codex-proxy-rs")
|
||||
self.root.path().join(BINARY_NAME)
|
||||
}
|
||||
|
||||
fn backup(&self) -> PathBuf {
|
||||
self.root.path().join(format!("{BINARY_NAME}.backup"))
|
||||
}
|
||||
|
||||
fn web(&self) -> PathBuf {
|
||||
@@ -393,7 +403,7 @@ impl Fixture {
|
||||
#[derive(Clone, Copy)]
|
||||
enum ArchiveKind {
|
||||
Safe,
|
||||
UnsafePath,
|
||||
UnsafePath(&'static str),
|
||||
MissingOfficialManifest,
|
||||
NestedOfficialAsset,
|
||||
OfficialSymlink,
|
||||
@@ -409,7 +419,7 @@ enum ChecksumKind {
|
||||
fn release_archive(kind: ArchiveKind) -> Vec<u8> {
|
||||
let encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
let mut tar = Builder::new(encoder);
|
||||
append_file(&mut tar, "codex-proxy-rs", b"new-binary", false);
|
||||
append_file(&mut tar, BINARY_NAME, b"new-binary", false);
|
||||
append_file(&mut tar, "web/dist/index.html", b"new-web", false);
|
||||
if !matches!(kind, ArchiveKind::MissingOfficialManifest) {
|
||||
append_file(
|
||||
@@ -425,8 +435,8 @@ fn release_archive(kind: ArchiveKind) -> Vec<u8> {
|
||||
b"new-plugin",
|
||||
false,
|
||||
);
|
||||
if matches!(kind, ArchiveKind::UnsafePath) {
|
||||
append_file(&mut tar, "safe", b"escape", true);
|
||||
if let ArchiveKind::UnsafePath(name) = kind {
|
||||
append_file(&mut tar, name, b"escape", true);
|
||||
}
|
||||
if matches!(kind, ArchiveKind::NestedOfficialAsset) {
|
||||
append_file(
|
||||
@@ -461,13 +471,19 @@ fn append_symlink(tar: &mut Builder<GzEncoder<Vec<u8>>>, name: &str, target: &st
|
||||
fn append_file(tar: &mut Builder<GzEncoder<Vec<u8>>>, name: &str, data: &[u8], unsafe_path: bool) {
|
||||
let mut header = Header::new_gnu();
|
||||
header.set_entry_type(EntryType::Regular);
|
||||
header.set_mode(0o755);
|
||||
header.set_mode(if name.starts_with("plugins/official/") {
|
||||
0o444
|
||||
} else {
|
||||
0o755
|
||||
});
|
||||
header.set_size(u64::try_from(data.len()).expect("size"));
|
||||
header.set_path(name).expect("path");
|
||||
header
|
||||
.set_path(if unsafe_path { "safe" } else { name })
|
||||
.expect("path");
|
||||
if unsafe_path {
|
||||
let bytes = header.as_mut_bytes();
|
||||
bytes[..100].fill(0);
|
||||
bytes[..9].copy_from_slice(b"../escape");
|
||||
bytes[..name.len()].copy_from_slice(name.as_bytes());
|
||||
}
|
||||
header.set_cksum();
|
||||
tar.append(&header, data).expect("append");
|
||||
|
||||
@@ -2,6 +2,120 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_and_rollback_preserve_a_running_executable() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fs::copy(
|
||||
env!("CARGO_BIN_EXE_gateway-host-process-worker"),
|
||||
fixture.executable(),
|
||||
)
|
||||
.expect("running executable");
|
||||
let original = fs::read(fixture.executable()).expect("original executable");
|
||||
fs::write(fixture.root.path().join("mode"), b"update-probe").expect("worker mode");
|
||||
let mut child = tokio::process::Command::new(fixture.executable())
|
||||
.current_dir(fixture.root.path())
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.expect("start old executable");
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
child.stdout.as_mut().unwrap().read_u8()
|
||||
)
|
||||
.await
|
||||
.expect("worker ready")
|
||||
.expect("ready byte"),
|
||||
b'R'
|
||||
);
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::Safe,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
let service = fixture.service(&server);
|
||||
let status = complete_update(&service, TARGET_VERSION).await;
|
||||
assert_eq!(
|
||||
status.operation.status,
|
||||
SystemOperationStatus::Succeeded,
|
||||
"{status:?}"
|
||||
);
|
||||
assert!(status.need_restart);
|
||||
assert_eq!(fs::read(fixture.backup()).expect("backup"), original);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("installed"),
|
||||
b"new-binary"
|
||||
);
|
||||
assert!(child.try_wait().expect("old process state").is_none());
|
||||
|
||||
service
|
||||
.rollback(Arc::new(AllowingUpdatePreflight))
|
||||
.await
|
||||
.expect("rollback running executable");
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("restored executable"),
|
||||
original
|
||||
);
|
||||
assert!(
|
||||
!service
|
||||
.update_status()
|
||||
.await
|
||||
.expect("restored status")
|
||||
.need_restart
|
||||
);
|
||||
|
||||
service
|
||||
.perform_update(
|
||||
Some(TARGET_VERSION.to_owned()),
|
||||
None,
|
||||
Arc::new(ChangingRevisionPreflight {
|
||||
confirmations: AtomicUsize::new(0),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("second update accepted");
|
||||
let failed = wait_for_update(&service).await;
|
||||
assert_eq!(
|
||||
failed.operation.status,
|
||||
SystemOperationStatus::Failed,
|
||||
"{failed:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("restored after failure"),
|
||||
original
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.web().join("index.html")).expect("restored web"),
|
||||
b"old-web"
|
||||
);
|
||||
|
||||
child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_u8(b'X')
|
||||
.await
|
||||
.expect("release old process");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(5), child.wait())
|
||||
.await
|
||||
.expect("worker exited")
|
||||
.expect("exit status")
|
||||
.success()
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.root.path().join("unblocked")).expect("old process result"),
|
||||
b"still-running"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_should_not_shutdown_when_replacement_spawn_fails() {
|
||||
let fixture = Fixture::new();
|
||||
@@ -149,7 +263,12 @@ async fn assert_system_operations_rejected(service: &ProcessSystemOperations, re
|
||||
async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
let fixture = Fixture::new();
|
||||
let replacement_marker = fixture.root.path().join("replacement-ran");
|
||||
fixture.write_executable("#!/bin/sh\n: > \"${0%/*}/replacement-ran\"\n");
|
||||
fs::copy(
|
||||
env!("CARGO_BIN_EXE_gateway-host-process-worker"),
|
||||
fixture.executable(),
|
||||
)
|
||||
.expect("replacement executable");
|
||||
fs::write(fixture.root.path().join("restart-probe"), b"").expect("restart probe");
|
||||
let mut config = fixture.config("http://127.0.0.1:1/repos");
|
||||
config.self_restart_enabled = true;
|
||||
let shutdown = CancellationToken::new();
|
||||
@@ -166,6 +285,21 @@ async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
|
||||
tokio::time::timeout(Duration::from_secs(2), shutdown.cancelled())
|
||||
.await
|
||||
.expect("shutdown requested after spawn");
|
||||
#[cfg(windows)]
|
||||
{
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !fixture.root.path().join("replacement-waiting").exists() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replacement waiting for old service");
|
||||
assert!(
|
||||
!replacement_marker.exists(),
|
||||
"new service must wait for resource release"
|
||||
);
|
||||
}
|
||||
drop(service);
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !replacement_marker.is_file() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
@@ -189,7 +323,7 @@ async fn restart_should_use_startup_path_after_running_executable_is_renamed() {
|
||||
let output = std::process::Command::new(child)
|
||||
.args([
|
||||
"--exact",
|
||||
"system_update::restart_should_use_startup_path_after_running_executable_is_renamed",
|
||||
"system_update::operations::restart_should_use_startup_path_after_running_executable_is_renamed",
|
||||
])
|
||||
.env(CHILD_ENV, "1")
|
||||
.env("CPR_RESTART_DELAY_MS", "1200")
|
||||
@@ -223,7 +357,7 @@ async fn restart_should_use_startup_path_after_running_executable_is_renamed() {
|
||||
config.self_restart_enabled = true;
|
||||
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
|
||||
|
||||
let backup = fixture.root.path().join("codex-proxy-rs.backup");
|
||||
let backup = fixture.backup();
|
||||
fs::rename(fixture.executable(), &backup).expect("back up running executable");
|
||||
fixture.write_executable("#!/bin/sh\nprintf new > \"${0%/*}/replacement-version\"\n");
|
||||
assert_eq!(std::env::current_exe().expect("renamed executable"), backup);
|
||||
@@ -777,24 +911,42 @@ async fn update_should_reject_release_archive_from_untrusted_host() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_reject_release_archive_with_unsafe_path() {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::UnsafePath,
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
for unsafe_path in [
|
||||
"../escape",
|
||||
#[cfg(windows)]
|
||||
"web/dist/index.html:payload",
|
||||
#[cfg(windows)]
|
||||
"web/dist/NUL.txt",
|
||||
#[cfg(windows)]
|
||||
"web/dist/COM¹",
|
||||
#[cfg(windows)]
|
||||
"web/dist/trailing. ",
|
||||
#[cfg(windows)]
|
||||
"C:/escape",
|
||||
] {
|
||||
let server = MockServer::start().await;
|
||||
let fixture = Fixture::new();
|
||||
fixture
|
||||
.mount_release(
|
||||
&server,
|
||||
TARGET_VERSION,
|
||||
ArchiveKind::UnsafePath(unsafe_path),
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
.await
|
||||
.operation
|
||||
.status,
|
||||
SystemOperationStatus::Failed,
|
||||
);
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
.await
|
||||
.operation
|
||||
.status,
|
||||
SystemOperationStatus::Failed,
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(fixture.executable()).expect("unchanged binary"),
|
||||
b"old-binary"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -933,6 +1085,7 @@ async fn update_and_rollback_should_use_the_default_api_asset_directory() {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn update_should_replace_web_assets_across_filesystems() {
|
||||
let Ok(external) = tempfile::tempdir_in("/dev/shm") else {
|
||||
@@ -980,7 +1133,7 @@ async fn update_should_restore_web_assets_when_binary_backup_fails() {
|
||||
ChecksumKind::Valid,
|
||||
)
|
||||
.await;
|
||||
fs::create_dir(fixture.executable().with_extension("backup")).expect("blocking backup dir");
|
||||
fs::create_dir(fixture.backup()).expect("blocking backup dir");
|
||||
|
||||
assert_eq!(
|
||||
complete_update(&fixture.service(&server), TARGET_VERSION)
|
||||
|
||||
@@ -145,6 +145,7 @@ pub struct RuntimeSettings {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -180,6 +181,7 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub max_retry_after_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
@@ -377,6 +379,7 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
max_retry_after_seconds: settings.max_retry_after_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
|
||||
@@ -104,6 +104,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
"max_waiting_per_key".to_owned(),
|
||||
"max_waiting_per_account".to_owned(),
|
||||
"concurrency_wait_timeout_seconds".to_owned(),
|
||||
"max_retry_after_seconds".to_owned(),
|
||||
"openai_guardian_reserved_concurrency".to_owned(),
|
||||
"openai_account_affinity".to_owned(),
|
||||
"max_account_rotations".to_owned(),
|
||||
|
||||
@@ -327,6 +327,7 @@ pub(crate) fn usage_list_record_from_row(
|
||||
provider_account_name: get(row, "provider_account_name")?,
|
||||
provider_account_email: get(row, "provider_account_email")?,
|
||||
provider_account_notes: get(row, "provider_account_notes")?,
|
||||
provider_account_upstream_peer: None,
|
||||
provider_account_plan_type: get(row, "provider_account_plan_type")?,
|
||||
provider_account_plan_type_display: None,
|
||||
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
|
||||
|
||||
@@ -781,7 +781,12 @@ impl ProxyStore for PgProxyRepository {
|
||||
.bind(command.enabled).bind(command.max_websocket_connections.map(i64::from))
|
||||
.execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?;
|
||||
if changed.rows_affected() != 1 {
|
||||
return Err(store_error(conflict(&command.id)));
|
||||
// 返回版本冲突前释放事务锁，避免误挡紧接着的导入预留
|
||||
let error = match transaction.rollback().await {
|
||||
Ok(()) => conflict(&command.id),
|
||||
Err(cleanup) => conflict(&command.id).with_cleanup(cleanup),
|
||||
};
|
||||
return Err(store_error(error));
|
||||
}
|
||||
if let Some(location) = &command.location {
|
||||
save_location(&mut transaction, &command.id, location.as_ref())
|
||||
@@ -852,7 +857,12 @@ impl ProxyStore for PgProxyRepository {
|
||||
.await
|
||||
.map_err(|source| store_error(unavailable(source)))?;
|
||||
if locked.is_none() {
|
||||
return Err(store_error(conflict(id)));
|
||||
// 返回版本冲突前释放事务锁，避免误挡紧接着的删除或编辑
|
||||
let error = match transaction.rollback().await {
|
||||
Ok(()) => conflict(id),
|
||||
Err(cleanup) => conflict(id).with_cleanup(cleanup),
|
||||
};
|
||||
return Err(store_error(error));
|
||||
}
|
||||
let account_ids: Vec<String> = sqlx::query_scalar(
|
||||
"update provider_accounts set outbound_proxy_id = null, outbound_proxy_url = null, provider_quota_json = null, quota_observed_at = null,
|
||||
|
||||
@@ -155,7 +155,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
|
||||
refresh_concurrency, max_concurrent_per_account,
|
||||
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
from runtime_settings where id = 1",
|
||||
@@ -311,7 +311,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
refresh_concurrency, max_concurrent_per_account,
|
||||
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
from runtime_settings where id = 1",
|
||||
@@ -391,6 +391,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
codex_privacy_policy_json = $28,
|
||||
reset_credit_lead_minutes = $29,
|
||||
reset_credit_max_weekly_remaining_percent = $30,
|
||||
max_retry_after_seconds = $31,
|
||||
updated_at = now()
|
||||
where id = 1
|
||||
returning config_revision",
|
||||
@@ -433,6 +434,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
.bind(sqlx::types::Json(&update.values.codex_privacy_policy))
|
||||
.bind(i64::from(update.values.reset_credit_lead_minutes))
|
||||
.bind(update.values.reset_credit_max_weekly_remaining_percent.map(i64::from))
|
||||
.bind(i64::from(update.values.max_retry_after_seconds))
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("update runtime settings in transaction", source))?
|
||||
@@ -520,6 +522,7 @@ struct RuntimeSettingsRow {
|
||||
max_waiting_per_key: i64,
|
||||
max_waiting_per_account: i64,
|
||||
concurrency_wait_timeout_seconds: i64,
|
||||
max_retry_after_seconds: i64,
|
||||
openai_guardian_reserved_concurrency: i64,
|
||||
openai_account_affinity: String,
|
||||
max_account_rotations: i64,
|
||||
@@ -578,6 +581,7 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
|
||||
max_waiting_per_key: to_u32(row.max_waiting_per_key)?,
|
||||
max_waiting_per_account: to_u32(row.max_waiting_per_account)?,
|
||||
concurrency_wait_timeout_seconds: to_u32(row.concurrency_wait_timeout_seconds)?,
|
||||
max_retry_after_seconds: to_u32(row.max_retry_after_seconds)?,
|
||||
openai_guardian_reserved_concurrency: to_u32(row.openai_guardian_reserved_concurrency)?,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::parse(
|
||||
&row.openai_account_affinity,
|
||||
|
||||
@@ -228,6 +228,7 @@ struct SnapshotSettingsRow {
|
||||
max_waiting_per_key: i64,
|
||||
max_waiting_per_account: i64,
|
||||
concurrency_wait_timeout_seconds: i64,
|
||||
max_retry_after_seconds: i64,
|
||||
openai_guardian_reserved_concurrency: i64,
|
||||
openai_account_affinity: String,
|
||||
max_account_rotations: i64,
|
||||
@@ -245,7 +246,7 @@ async fn load_settings(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> StoreResult<(Revision, SettingsValues)> {
|
||||
let row = sqlx::query_as::<_, SnapshotSettingsRow>(
|
||||
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
|
||||
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
@@ -275,6 +276,7 @@ async fn load_settings(
|
||||
.ok_or_else(|| invalid("invalid account affinity"))?,
|
||||
)
|
||||
.with_max_account_rotations(to_u32(row.max_account_rotations)?)
|
||||
.with_max_retry_after_seconds(to_u32(row.max_retry_after_seconds)?)
|
||||
.with_openai_session_affinity_ttl_hours(to_u32(row.openai_session_affinity_ttl_hours)?)
|
||||
.with_smart_scheduling(row.smart_scheduling_json.0)
|
||||
.with_request_profiles(decode_request_profiles(
|
||||
|
||||
@@ -87,17 +87,13 @@ async fn slow_settlement_does_not_block_receipt_or_concurrent_admission() {
|
||||
let worker_context = WorkerDaemonContext::new(CancellationToken::new());
|
||||
let running_context = worker_context.clone();
|
||||
let task = tokio::spawn(async move { writer.run(running_context).await });
|
||||
tokio::time::timeout(Duration::from_millis(500), async {
|
||||
for result in
|
||||
futures::future::join_all((0..4).map(|_| store.admit(key_id("allowed")))).await
|
||||
{
|
||||
result.unwrap();
|
||||
}
|
||||
let error = store.admit(key_id("exhausted")).await.unwrap_err();
|
||||
assert_eq!(error.client_error_code(), Some("key_daily_budget_exceeded"));
|
||||
})
|
||||
.await
|
||||
.expect("admission must not wait for settlement row locks");
|
||||
// 行锁直到准入断言结束才释放，等待行锁会被准入自身的一秒期限判为失败
|
||||
// 不给整组请求另设更短的期限，避免把连接建立和并发调度耗时误判为锁等待
|
||||
for result in futures::future::join_all((0..4).map(|_| store.admit(key_id("allowed")))).await {
|
||||
result.expect("admission must not wait for settlement row locks");
|
||||
}
|
||||
let error = store.admit(key_id("exhausted")).await.unwrap_err();
|
||||
assert_eq!(error.client_error_code(), Some("key_daily_budget_exceeded"));
|
||||
let before: i64 = sqlx::query_scalar("select count(*) from client_api_key_charge_events")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
|
||||
@@ -333,7 +333,12 @@ async fn disabled_proxy_projects_direct_while_preserving_bindings_and_credential
|
||||
);
|
||||
assert_eq!(proxies.get(&saved.id).await.unwrap().enabled, enabled);
|
||||
let reservation = proxies.reserve_import(&saved.id).await;
|
||||
assert_eq!(reservation.is_ok(), enabled);
|
||||
assert_eq!(
|
||||
reservation.is_ok(),
|
||||
enabled,
|
||||
"import reservation failed: {:?}",
|
||||
reservation.as_ref().err()
|
||||
);
|
||||
drop(reservation);
|
||||
if !enabled {
|
||||
let error = admin
|
||||
@@ -730,8 +735,8 @@ async fn rejected_import_reservations_release_proxy_lock_before_returning() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_proxy_test_releases_lock_before_next_update() {
|
||||
let Some(database) = TestDatabase::create("stale_proxy_test_lock").await else {
|
||||
async fn stale_proxy_mutations_release_locks_before_next_update() {
|
||||
let Some(database) = TestDatabase::create("stale_proxy_mutation_lock").await else {
|
||||
return;
|
||||
};
|
||||
let store = PgProxyRepository::new(database.pool.clone());
|
||||
@@ -757,52 +762,63 @@ async fn stale_proxy_test_releases_lock_before_next_update() {
|
||||
.unwrap()
|
||||
.record;
|
||||
|
||||
// 暂停连接归还时的后台清理，确保锁由返回错误前的显式回滚释放
|
||||
let release_gate = std::sync::Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let rejected_pool = database
|
||||
.pool
|
||||
.options()
|
||||
.clone()
|
||||
.max_connections(1)
|
||||
.after_release({
|
||||
let release_gate = release_gate.clone();
|
||||
move |_, _| {
|
||||
for mutation in ["record_test", "update", "delete"] {
|
||||
// 暂停连接归还时的后台清理，确保锁由返回错误前的显式回滚释放
|
||||
let release_gate = std::sync::Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let rejected_pool = database
|
||||
.pool
|
||||
.options()
|
||||
.clone()
|
||||
.max_connections(1)
|
||||
.after_release({
|
||||
let release_gate = release_gate.clone();
|
||||
Box::pin(async move {
|
||||
let _permit = release_gate.acquire().await.unwrap();
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
})
|
||||
.connect_with((*database.pool.connect_options()).clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let stale_writer = PgProxyRepository::new(rejected_pool.clone());
|
||||
let error = stale_writer
|
||||
.record_test(&saved.id, stale_revision, success(), &context)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let update = store
|
||||
.update(
|
||||
UpdateProxy {
|
||||
max_websocket_connections: None,
|
||||
enabled: None,
|
||||
id: saved.id.clone(),
|
||||
revision: saved.revision,
|
||||
name: saved.name.clone(),
|
||||
proxy: None,
|
||||
auto_location: None,
|
||||
location: None,
|
||||
test: None,
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.await;
|
||||
release_gate.add_permits(1);
|
||||
rejected_pool.close().await;
|
||||
move |_, _| {
|
||||
let release_gate = release_gate.clone();
|
||||
Box::pin(async move {
|
||||
let _permit = release_gate.acquire().await.unwrap();
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
})
|
||||
.connect_with((*database.pool.connect_options()).clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let stale_writer = PgProxyRepository::new(rejected_pool.clone());
|
||||
let command = |revision| UpdateProxy {
|
||||
max_websocket_connections: None,
|
||||
enabled: None,
|
||||
id: saved.id.clone(),
|
||||
revision,
|
||||
name: saved.name.clone(),
|
||||
proxy: None,
|
||||
auto_location: None,
|
||||
location: None,
|
||||
test: None,
|
||||
};
|
||||
let error = match mutation {
|
||||
"record_test" => stale_writer
|
||||
.record_test(&saved.id, stale_revision, success(), &context)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
"update" => stale_writer
|
||||
.update(command(stale_revision), &context)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
"delete" => stale_writer
|
||||
.delete(&saved.id, stale_revision, &context)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let update = store.update(command(saved.revision), &context).await;
|
||||
release_gate.add_permits(1);
|
||||
rejected_pool.close().await;
|
||||
assert_eq!(error.kind(), AdminStoreErrorKind::Conflict);
|
||||
saved = update
|
||||
.unwrap_or_else(|error| panic!("过期 {mutation} 不应阻挡后续编辑: {error:?}"))
|
||||
.record;
|
||||
}
|
||||
database.close().await;
|
||||
assert_eq!(error.kind(), AdminStoreErrorKind::Conflict);
|
||||
assert!(update.is_ok(), "过期检测结果不应阻挡后续编辑: {update:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -31,6 +31,7 @@ pub(super) fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettin
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
max_retry_after_seconds: 240,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
@@ -504,6 +505,56 @@ fn timestamp_micros(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
DateTime::from_timestamp_micros(value.timestamp_micros()).expect("valid test timestamp")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_after_limit_defaults_and_updates_reach_the_runtime_snapshot() {
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create("retry_after_limit").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.max_retry_after_seconds,
|
||||
240
|
||||
);
|
||||
for limit in [60, 0, u32::MAX] {
|
||||
let mut update = settings_with_margin(300);
|
||||
update.values.max_retry_after_seconds = limit;
|
||||
repository.update_runtime_settings(update).await.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.max_retry_after_seconds,
|
||||
limit
|
||||
);
|
||||
let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone())
|
||||
.load_runtime_snapshot()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(snapshot.settings).unwrap()["max_retry_after_seconds"],
|
||||
limit
|
||||
);
|
||||
}
|
||||
for invalid in [-1_i64, i64::from(u32::MAX) + 1] {
|
||||
assert!(
|
||||
sqlx::query("update runtime_settings set max_retry_after_seconds = $1 where id = 1")
|
||||
.bind(invalid)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
|
||||
@@ -4,12 +4,14 @@
|
||||
mod headers;
|
||||
mod request;
|
||||
mod search_history;
|
||||
mod search_tools;
|
||||
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub(crate) use headers::is_non_codex_request_header;
|
||||
pub(crate) use request::normalize_codex_request_body;
|
||||
pub(crate) use search_tools::declares_hosted_search;
|
||||
|
||||
/// 选定 Codex/OAuth 上游后处理客户端正文兼容
|
||||
pub(crate) fn normalize_selected_codex_downstream_body(
|
||||
|
||||
@@ -4,6 +4,8 @@ use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_protocol::openai::X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::search_tools::tools_declare_search;
|
||||
|
||||
// TODO(https://github.com/openai/codex/issues/51962): 官方修复历史搜索项与工具声明不匹配后，
|
||||
// 用含 web_search_call 的普通请求和 Lite 压缩请求验证直连均可完成，再移除此补偿及对应合同测试
|
||||
// Codex 0.163.0 本地压缩可保留搜索历史却使用空工具；真实上游对照中补声明可消除相应 500
|
||||
@@ -51,7 +53,7 @@ fn normalize_body(body: &mut Map<String, Value>, lite: bool) -> bool {
|
||||
Some(Value::Array(tools)) => tools.as_slice(),
|
||||
_ => return false,
|
||||
};
|
||||
if declares_search(tools) {
|
||||
if tools_declare_search(tools) {
|
||||
return false;
|
||||
}
|
||||
let mut has_tools = !tools.is_empty();
|
||||
@@ -64,7 +66,7 @@ fn normalize_body(body: &mut Map<String, Value>, lite: bool) -> bool {
|
||||
// 不修补非法工具结构，保留上游原有校验
|
||||
return false;
|
||||
};
|
||||
if declares_search(tools) {
|
||||
if tools_declare_search(tools) {
|
||||
return false;
|
||||
}
|
||||
has_tools |= !tools.is_empty();
|
||||
@@ -105,11 +107,3 @@ fn normalize_body(body: &mut Map<String, Value>, lite: bool) -> bool {
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn declares_search(tools: &[Value]) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
tool.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|kind| kind == "web_search" || kind.starts_with("web_search_preview"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
//! 识别普通与 Responses Lite 请求中的托管搜索声明，不改写工具配置
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub(crate) fn declares_hosted_search(body: &Map<String, Value>) -> bool {
|
||||
body.get("tools")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|tools| tools_declare_search(tools))
|
||||
|| body
|
||||
.get("input")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|input| {
|
||||
input.iter().any(|item| {
|
||||
matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("additional_tools" | "tool_search_output")
|
||||
) && item
|
||||
.get("tools")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|tools| tools_declare_search(tools))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn tools_declare_search(tools: &[Value]) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
tool.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|kind| kind == "web_search" || kind.starts_with("web_search_preview"))
|
||||
})
|
||||
}
|
||||
@@ -397,6 +397,7 @@ pub(super) struct ColdResponse {
|
||||
pub(super) session_transport_key_hash: Option<String>,
|
||||
pub(super) session_transport_recovery: CodexSessionTransportRecovery,
|
||||
pub(super) guardian_routing: guardian::GuardianRouting,
|
||||
pub(super) turn_state_scope: Option<turn_state::TurnStateScope>,
|
||||
pub(super) websocket_retry_count: u32,
|
||||
pub(super) stream_max_retries: u32,
|
||||
pub(super) session_capture: Option<OpenAiSessionCapture>,
|
||||
@@ -784,6 +785,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
session_transport_key_hash,
|
||||
session_transport_recovery,
|
||||
guardian_routing,
|
||||
turn_state_scope,
|
||||
websocket_retry_count,
|
||||
stream_max_retries,
|
||||
mut session_capture,
|
||||
@@ -873,6 +875,17 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(scope) = &turn_state_scope
|
||||
&& let Some(response) = failure.error.client_visible_upstream_response()
|
||||
{
|
||||
for header in response.headers() {
|
||||
if header.name().eq_ignore_ascii_case("x-codex-turn-state")
|
||||
&& let Ok(token) = std::str::from_utf8(header.value().as_ref())
|
||||
{
|
||||
scope.remember(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(observation) = failure.observation.take() {
|
||||
yield ProviderEvent::observation(observation);
|
||||
}
|
||||
@@ -897,6 +910,11 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
Err(failure.error)?;
|
||||
return;
|
||||
}
|
||||
if let Some(scope) = &turn_state_scope
|
||||
&& let Some(token) = response.turn_state.as_deref()
|
||||
{
|
||||
scope.remember(token);
|
||||
}
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.continuation_scope = Some(if capture.response_store {
|
||||
OpenAiContinuationScope::Persisted
|
||||
@@ -1040,6 +1058,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
response_metadata_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
turn_state_scope.as_ref(),
|
||||
)
|
||||
.await;
|
||||
let observation_event = if rate_limits_changed || metadata_merge.is_some()
|
||||
@@ -1117,6 +1136,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
response_metadata_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
turn_state_scope.as_ref(),
|
||||
)
|
||||
.await;
|
||||
let metadata_changed = metadata_merge.unwrap_or(false);
|
||||
@@ -1317,6 +1337,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
response_metadata_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
turn_state_scope.as_ref(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
@@ -1384,8 +1405,12 @@ async fn merge_response_metadata_updates(
|
||||
updates: Option<&CodexResponseMetadataUpdates>,
|
||||
session_capture: &mut Option<OpenAiSessionCapture>,
|
||||
observation_state: &mut OpenAiResponseObservationState,
|
||||
turn_state_scope: Option<&turn_state::TurnStateScope>,
|
||||
) -> Option<bool> {
|
||||
let turn_state = updates?.lock().await.turn_state.take()?;
|
||||
if let Some(scope) = turn_state_scope {
|
||||
scope.remember(&turn_state);
|
||||
}
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.turn_state = Some(turn_state.clone());
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ mod failure;
|
||||
mod guardian;
|
||||
mod live;
|
||||
mod observation;
|
||||
mod turn_state;
|
||||
mod upstream_adapter;
|
||||
mod workers;
|
||||
pub(crate) use live::{CodexLiveGateway, CodexLiveRegistry};
|
||||
@@ -156,6 +157,7 @@ pub struct CodexProvider {
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_recovery: CodexSessionTransportRecovery,
|
||||
guardian_routing: guardian::GuardianRouting,
|
||||
turn_state_ownership: turn_state::TurnStateOwnership,
|
||||
stream_max_retries: u32,
|
||||
live_registry: Arc<CodexLiveRegistry>,
|
||||
live_gateway: Option<Arc<CodexLiveGateway>>,
|
||||
@@ -361,6 +363,7 @@ impl CodexProvider {
|
||||
session_identity: None,
|
||||
session_transport_recovery: CodexSessionTransportRecovery::default(),
|
||||
guardian_routing: guardian::GuardianRouting::default(),
|
||||
turn_state_ownership: turn_state::TurnStateOwnership::default(),
|
||||
stream_max_retries,
|
||||
live_registry: Arc::new(CodexLiveRegistry::default()),
|
||||
live_gateway: None,
|
||||
@@ -963,10 +966,25 @@ impl CodexProvider {
|
||||
)
|
||||
.map(|affinity| (response_id.to_owned(), affinity))
|
||||
});
|
||||
let turn_state_scope = self
|
||||
.turn_state_ownership
|
||||
.scope(&context, &lease, &upstream_request);
|
||||
// 全量请求的 token 证明不能用于放行 response ID 或其他账号状态
|
||||
let verified_turn_state = upstream_request
|
||||
.turn_state
|
||||
.as_ref()
|
||||
.filter(|token| {
|
||||
matches!(account_scope, RequestAccountScope::Unknown)
|
||||
&& turn_state_scope
|
||||
.as_ref()
|
||||
.is_some_and(|scope| scope.owns(token))
|
||||
})
|
||||
.cloned();
|
||||
scope_request_to_account(
|
||||
&mut upstream_request,
|
||||
lease.installation_id(),
|
||||
account_scope,
|
||||
verified_turn_state.as_deref(),
|
||||
);
|
||||
// 父响应的可信关联只恢复此字段，不把首次 Guardian 请求的其他账号状态视为可信
|
||||
if let Some((response_id, _)) = &guardian_parent
|
||||
@@ -989,6 +1007,7 @@ impl CodexProvider {
|
||||
"ownerCrossAccount": state_owner_cross_account,
|
||||
"accountSwitch": lease.account_switch(),
|
||||
"accountStateReusable": account_scope.can_reuse_account_state(),
|
||||
"verifiedTurnStateEcho": verified_turn_state.is_some(),
|
||||
"before": before,
|
||||
"after": crate::transport::diagnostics::evidence::request_state(&upstream_request),
|
||||
}));
|
||||
@@ -1162,6 +1181,7 @@ impl CodexProvider {
|
||||
session_transport_key_hash,
|
||||
session_transport_recovery: self.session_transport_recovery.clone(),
|
||||
guardian_routing: self.guardian_routing.clone(),
|
||||
turn_state_scope,
|
||||
websocket_retry_count,
|
||||
stream_max_retries: self.stream_max_retries,
|
||||
session_capture,
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! 只为上游已返回的 turn state 保存归属证明，不恢复正文或参与账号选择
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use gateway_core::engine::AttemptContext;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
credential::{CodexCredentialLease, CodexRuntimeAuthentication},
|
||||
transport::protocol::responses::CodexResponsesRequest,
|
||||
};
|
||||
|
||||
// 仅保存 32 字节摘要与过期时间，4096 项的条目数据约 192 KiB，不随 token 长度增长
|
||||
const MAX_PROOFS: usize = 4096;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct TurnStateOwnership {
|
||||
proofs: Arc<Mutex<HashMap<[u8; 32], Instant>>>,
|
||||
}
|
||||
|
||||
pub(super) struct TurnStateScope {
|
||||
ownership: TurnStateOwnership,
|
||||
identity: Sha256,
|
||||
retention: Duration,
|
||||
}
|
||||
|
||||
impl TurnStateOwnership {
|
||||
pub(super) fn scope(
|
||||
&self,
|
||||
context: &AttemptContext,
|
||||
lease: &CodexCredentialLease,
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<TurnStateScope> {
|
||||
let turn_id = request
|
||||
.client_turn_id
|
||||
.as_deref()
|
||||
.filter(|id| !id.is_empty())?;
|
||||
let mut identity = Sha256::new();
|
||||
for part in [
|
||||
context.client_api_key_ref().as_str(),
|
||||
lease.account_id().as_str(),
|
||||
request.client_session_id.as_deref().unwrap_or_default(),
|
||||
request.client_thread_id.as_deref().unwrap_or_default(),
|
||||
turn_id,
|
||||
] {
|
||||
identity.update(part.len().to_le_bytes());
|
||||
identity.update(part.as_bytes());
|
||||
}
|
||||
// OAuth 刷新不改变账号归属；API Key 替换则不能继承原认证的路由状态
|
||||
if matches!(
|
||||
lease.authentication(),
|
||||
CodexRuntimeAuthentication::ApiKey(_)
|
||||
) {
|
||||
identity.update([1]);
|
||||
identity.update(lease.account().revision().get().to_le_bytes());
|
||||
} else {
|
||||
identity.update([0]);
|
||||
}
|
||||
Some(TurnStateScope {
|
||||
ownership: self.clone(),
|
||||
identity,
|
||||
retention: context
|
||||
.account_selection_policy()
|
||||
.openai_session_affinity_ttl(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TurnStateScope {
|
||||
pub(super) fn owns(&self, token: &str) -> bool {
|
||||
let key = self.key(token);
|
||||
let mut proofs = self
|
||||
.ownership
|
||||
.proofs
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(expires_at) = proofs.get_mut(&key) else {
|
||||
return false;
|
||||
};
|
||||
let now = Instant::now();
|
||||
if *expires_at <= now {
|
||||
proofs.remove(&key);
|
||||
return false;
|
||||
}
|
||||
*expires_at = now + self.retention;
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn remember(&self, token: &str) {
|
||||
if token.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = self.key(token);
|
||||
let now = Instant::now();
|
||||
let mut proofs = self
|
||||
.ownership
|
||||
.proofs
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if proofs.len() >= MAX_PROOFS && !proofs.contains_key(&key) {
|
||||
proofs.retain(|_, expires_at| *expires_at > now);
|
||||
if proofs.len() >= MAX_PROOFS
|
||||
&& let Some(oldest) = proofs
|
||||
.iter()
|
||||
.min_by_key(|(_, expires_at)| **expires_at)
|
||||
.map(|(key, _)| *key)
|
||||
{
|
||||
proofs.remove(&oldest);
|
||||
}
|
||||
}
|
||||
proofs.insert(key, now + self.retention);
|
||||
}
|
||||
|
||||
fn key(&self, token: &str) -> [u8; 32] {
|
||||
let mut hash = self.identity.clone();
|
||||
hash.update(token.len().to_le_bytes());
|
||||
hash.update(token.as_bytes());
|
||||
hash.finalize().into()
|
||||
}
|
||||
}
|
||||
@@ -23,16 +23,6 @@ const ID_FIELDS: &[&str] = &[
|
||||
"parentThreadId",
|
||||
];
|
||||
const METADATA_FIELDS: &[&str] = &["turnMetadata", "turn_metadata", "x-codex-turn-metadata"];
|
||||
const TRACE_FIELDS: &[&str] = &[
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
"sentry-trace",
|
||||
"baggage",
|
||||
"ws_request_header_traceparent",
|
||||
"ws_request_header_tracestate",
|
||||
"ws_request_header_sentry-trace",
|
||||
"ws_request_header_baggage",
|
||||
];
|
||||
// Live 的 x-session-id 与 voice-session-id 是业务会话引用，必须原样传递
|
||||
const ID_HEADERS: &[&str] = &[
|
||||
"session-id",
|
||||
@@ -102,25 +92,14 @@ fn map_fields(object: &mut Map<String, Value>, installation_id: &str, fields: &[
|
||||
changed
|
||||
}
|
||||
|
||||
fn scope_object(object: &mut Map<String, Value>, installation_id: Option<&str>) -> usize {
|
||||
let mut changed = 0;
|
||||
for key in TRACE_FIELDS {
|
||||
changed += usize::from(object.remove(*key).is_some());
|
||||
}
|
||||
if let Some(installation_id) = installation_id {
|
||||
changed += map_fields(object, installation_id, ID_FIELDS);
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
fn scope_metadata(value: &mut Value, installation_id: Option<&str>) {
|
||||
fn scope_metadata(value: &mut Value, installation_id: &str) {
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
scope_object(object, installation_id);
|
||||
map_fields(object, installation_id, ID_FIELDS);
|
||||
}
|
||||
Value::String(raw) => {
|
||||
if let Ok(Value::Object(mut object)) = serde_json::from_str(raw)
|
||||
&& scope_object(&mut object, installation_id) > 0
|
||||
&& map_fields(&mut object, installation_id, ID_FIELDS) > 0
|
||||
&& let Some(encoded) = serialize_ascii_turn_metadata(&Value::Object(object))
|
||||
{
|
||||
*raw = encoded;
|
||||
@@ -131,29 +110,27 @@ fn scope_metadata(value: &mut Value, installation_id: Option<&str>) {
|
||||
}
|
||||
|
||||
pub(super) fn scope_headers(headers: &mut HeaderMap, installation_id: Option<&str>) {
|
||||
for name in TRACE_FIELDS {
|
||||
headers.remove(*name);
|
||||
}
|
||||
let Some(installation_id) = installation_id else {
|
||||
return;
|
||||
};
|
||||
for name in ID_HEADERS {
|
||||
if let Some(installation_id) = installation_id {
|
||||
let values: Vec<_> = headers
|
||||
.get_all(*name)
|
||||
.iter()
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
.and_then(|value| {
|
||||
HeaderValue::from_str(&scoped_id(installation_id, value)).ok()
|
||||
})
|
||||
.unwrap_or_else(|| value.clone())
|
||||
})
|
||||
.collect();
|
||||
headers.remove(*name);
|
||||
for value in values {
|
||||
headers.append(*name, value);
|
||||
}
|
||||
let values: Vec<_> = headers
|
||||
.get_all(*name)
|
||||
.iter()
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
.and_then(|value| {
|
||||
HeaderValue::from_str(&scoped_id(installation_id, value)).ok()
|
||||
})
|
||||
.unwrap_or_else(|| value.clone())
|
||||
})
|
||||
.collect();
|
||||
headers.remove(*name);
|
||||
for value in values {
|
||||
headers.append(*name, value);
|
||||
}
|
||||
}
|
||||
let values: Vec<_> = headers
|
||||
@@ -183,18 +160,19 @@ pub(super) fn scope_request(
|
||||
installation_id: Option<&str>,
|
||||
) {
|
||||
scope_headers(headers, installation_id);
|
||||
let Some(installation_id) = installation_id else {
|
||||
return;
|
||||
};
|
||||
if let Some(object) = body.as_object_mut() {
|
||||
scope_object(object, installation_id);
|
||||
if let Some(installation_id) = installation_id {
|
||||
map_fields(object, installation_id, &["prompt_cache_key"]);
|
||||
}
|
||||
map_fields(object, installation_id, ID_FIELDS);
|
||||
map_fields(object, installation_id, &["prompt_cache_key"]);
|
||||
for key in METADATA_FIELDS {
|
||||
if let Some(value) = object.get_mut(*key) {
|
||||
scope_metadata(value, installation_id);
|
||||
}
|
||||
}
|
||||
if let Some(Value::Object(metadata)) = object.get_mut("client_metadata") {
|
||||
scope_object(metadata, installation_id);
|
||||
map_fields(metadata, installation_id, ID_FIELDS);
|
||||
for key in METADATA_FIELDS {
|
||||
if let Some(value) = metadata.get_mut(*key) {
|
||||
scope_metadata(value, installation_id);
|
||||
@@ -202,9 +180,6 @@ pub(super) fn scope_request(
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(installation_id) = installation_id else {
|
||||
return;
|
||||
};
|
||||
// 只处理官方自动 Git 上下文的关联字段，不扫描普通用户输入和工具输出
|
||||
let Ok(targets) = locate_contexts(body, true, || Ok(())) else {
|
||||
return;
|
||||
|
||||
@@ -482,15 +482,22 @@ fn classify_upstream_failure(
|
||||
};
|
||||
}
|
||||
|
||||
// 参数值不支持属于请求错误，不能因消息提及模型而推断账号无模型权限
|
||||
// 官方 Codex 将流内 invalid_prompt 归为请求错误，不依赖 HTTP 状态码
|
||||
if matches!(source, UpstreamFailureSource::SseFailure) && code == "invalid_prompt" {
|
||||
if code == "unsupported_value"
|
||||
|| (matches!(source, UpstreamFailureSource::SseFailure) && code == "invalid_prompt")
|
||||
{
|
||||
return CodexFailureCategory::InvalidRequest;
|
||||
}
|
||||
|
||||
if [code.as_str(), message.as_str(), body.as_str()]
|
||||
.into_iter()
|
||||
.any(is_model_unsupported)
|
||||
{
|
||||
// 对齐官方 Codex 的模型拒绝合同，消息与正文不提供账号模型资格证据
|
||||
if matches!(
|
||||
(fields.code.as_deref(), status),
|
||||
(
|
||||
Some("model_not_found"),
|
||||
None | Some(StatusCode::BAD_REQUEST | StatusCode::FORBIDDEN | StatusCode::NOT_FOUND)
|
||||
)
|
||||
) {
|
||||
return CodexFailureCategory::ModelUnsupported;
|
||||
}
|
||||
let identity_signals = [
|
||||
@@ -594,16 +601,6 @@ fn non_empty_owned(value: &str) -> Option<String> {
|
||||
(!value.is_empty()).then(|| value.to_owned())
|
||||
}
|
||||
|
||||
fn is_model_unsupported(value: &str) -> bool {
|
||||
value.contains("model_not_supported")
|
||||
|| value.contains("model_not_available")
|
||||
|| (value.contains("model")
|
||||
&& (value.contains("not supported")
|
||||
|| value.contains("not available")
|
||||
|| value.contains("not_supported")
|
||||
|| value.contains("not_available")))
|
||||
}
|
||||
|
||||
fn is_identity_verification_required(value: &str) -> bool {
|
||||
matches!(
|
||||
value,
|
||||
|
||||
@@ -28,16 +28,15 @@ pub(super) fn is_managed_identity_header(name: &str) -> bool {
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
// 工作区路由和合规属性属于认证账号，不能继承下游账号的值
|
||||
| "x-openai-account-routing-override"
|
||||
| "x-openai-fedramp"
|
||||
// 安装身份由当前账号写入 client_metadata，不继承下游安装头
|
||||
// 安装头与 client_metadata 共用当前账号的安装身份
|
||||
| "x-codex-installation-id"
|
||||
// 画像和设备证明须属于选定账号，不能由下游覆盖
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
// 地域和账号路由统一按部署配置生成
|
||||
| "x-openai-internal-codex-residency"
|
||||
| "x-openai-account-routing-override"
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
@@ -127,6 +126,11 @@ impl CodexBackendClient {
|
||||
};
|
||||
let mut headers = build_codex_model_headers(profile, context.authorization, account_id)?;
|
||||
insert_optional_header(&mut headers, "cookie", cookie_header)?;
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
|
||||
@@ -469,19 +469,17 @@ pub(crate) fn scope_request_to_account(
|
||||
request: &mut CodexResponsesRequest,
|
||||
installation_id: &str,
|
||||
account_scope: RequestAccountScope,
|
||||
verified_turn_state: Option<&str>,
|
||||
) {
|
||||
let reset_account_state = !account_scope.can_reuse_account_state();
|
||||
let client_metadata_turn_state = metadata_string(request, "x-codex-turn-state");
|
||||
let preserve_turn_state = !reset_account_state
|
||||
&& (request.turn_state.is_some() || client_metadata_turn_state.is_some());
|
||||
let turn_state = preserve_turn_state
|
||||
.then(|| request.turn_state.clone())
|
||||
.flatten();
|
||||
let client_metadata_turn_state = if preserve_turn_state {
|
||||
client_metadata_turn_state
|
||||
let turn_state = if reset_account_state {
|
||||
verified_turn_state.map(ToOwned::to_owned)
|
||||
} else {
|
||||
None
|
||||
request.turn_state.clone()
|
||||
};
|
||||
let client_metadata_turn_state = client_metadata_turn_state
|
||||
.filter(|token| !reset_account_state || Some(token.as_str()) == verified_turn_state);
|
||||
let turn_metadata = request
|
||||
.turn_metadata
|
||||
.as_deref()
|
||||
|
||||
@@ -154,6 +154,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
.acquire(
|
||||
&key,
|
||||
required_response_id,
|
||||
request.codex_search_declared == Some(true),
|
||||
request.connection().outbound_proxy.as_ref(),
|
||||
downstream_cancellation,
|
||||
)
|
||||
@@ -504,6 +505,7 @@ async fn connect_websocket_connection(
|
||||
metadata: websocket_connection_metadata(&response, connection),
|
||||
continuation: WebSocketContinuationState::default(),
|
||||
created_at: tokio::time::Instant::now(),
|
||||
initial_search_declared: None,
|
||||
},
|
||||
started_at.elapsed(),
|
||||
))
|
||||
@@ -614,6 +616,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
metadata,
|
||||
continuation,
|
||||
created_at,
|
||||
initial_search_declared,
|
||||
} = connection;
|
||||
trace.record("upstream.connection", serde_json::json!({
|
||||
"connectionId": websocket.connection_id().to_string(), "reused": reused,
|
||||
@@ -660,6 +663,8 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
lease,
|
||||
created_at,
|
||||
continuation,
|
||||
initial_search_declared: initial_search_declared
|
||||
.unwrap_or(request.codex_search_declared.unwrap_or(false)),
|
||||
});
|
||||
let mut exchange = stream_websocket_response(
|
||||
websocket,
|
||||
|
||||
@@ -30,6 +30,7 @@ pub(in crate::transport::websocket) struct WebSocketStreamPoolReturn {
|
||||
pub(in crate::transport::websocket) lease: WebSocketPoolLease,
|
||||
pub(in crate::transport::websocket) created_at: tokio::time::Instant,
|
||||
pub(in crate::transport::websocket) continuation: WebSocketContinuationState,
|
||||
pub(in crate::transport::websocket) initial_search_declared: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -446,6 +447,7 @@ async fn finish_stream_websocket(
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
continuation,
|
||||
created_at: pool_return.created_at,
|
||||
initial_search_declared: Some(pool_return.initial_search_declared),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -128,6 +128,8 @@ impl CodexWebSocketConnection {
|
||||
connection,
|
||||
payload_text: websocket_response_create_payload_text(request)?,
|
||||
continuation: WebSocketContinuationRequirement::from_request(request),
|
||||
codex_search_declared: (path == CODEX_RESPONSES_PATH)
|
||||
.then(|| crate::compat::declares_hosted_search(request.body())),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,8 @@ pub struct CodexWebSocketRequest {
|
||||
pub(crate) connection: CodexWebSocketConnection,
|
||||
pub(super) payload_text: String,
|
||||
pub(super) continuation: WebSocketContinuationRequirement,
|
||||
// 仅 Codex backend 约束首包搜索初始化，API Key 的 Responses API 不采用此规则
|
||||
pub(super) codex_search_declared: Option<bool>,
|
||||
}
|
||||
|
||||
/// 当前 WebSocket 请求对 previous response 状态的要求
|
||||
|
||||
@@ -131,6 +131,7 @@ impl CodexWebSocketPool {
|
||||
&self,
|
||||
key: &CodexWebSocketPoolKey,
|
||||
required_response_id: Option<&str>,
|
||||
requires_search: bool,
|
||||
proxy: Option<&gateway_core::account::OutboundProxy>,
|
||||
downstream_cancellation: Option<&gateway_core::lifecycle::CancellationToken>,
|
||||
) -> WebSocketPoolAcquire {
|
||||
@@ -204,6 +205,18 @@ impl CodexWebSocketPool {
|
||||
.observation()
|
||||
.with_exit_reason("downstream_connection_closed")
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
// https://github.com/zyycn/codex-proxy-rs/issues/570
|
||||
// Codex 首包未初始化搜索时不能在原 socket 后加声明，generate=false 也占首包
|
||||
// 上游允许后续启用搜索并经直连验证后可移除此回收条件
|
||||
(requires_search && connection.initial_search_declared == Some(false))
|
||||
.then(|| {
|
||||
connection
|
||||
.websocket
|
||||
.observation()
|
||||
.with_exit_reason("search_activation_required")
|
||||
})
|
||||
});
|
||||
let Some(observation) = loss else {
|
||||
connection.websocket.revoke_control();
|
||||
|
||||
@@ -197,6 +197,8 @@ pub(crate) struct PooledWebSocketConnection {
|
||||
pub(crate) metadata: CodexWebSocketConnectionMetadata,
|
||||
pub(crate) continuation: WebSocketContinuationState,
|
||||
pub(crate) created_at: Instant,
|
||||
// 首个已写入请求决定搜索能力，None 表示仅完成握手，后续工具变化不能覆盖此事实
|
||||
pub(crate) initial_search_declared: Option<bool>,
|
||||
}
|
||||
|
||||
/// 只随具体 WebSocket 生命周期存在的续接状态
|
||||
|
||||
@@ -45,6 +45,7 @@ const DOWNSTREAM_CLIENT_HEADERS: &[(&str, &str)] = &[
|
||||
];
|
||||
|
||||
const BUSINESS_EXTENSION_HEADERS: &[(&str, &str)] = &[
|
||||
("x-openai-fedramp", "true"),
|
||||
("x-grok-turn-idx", "7"),
|
||||
("X-Grok-Model-Override", "synthetic-model"),
|
||||
("x-grok-session-id", "synthetic-grok-session"),
|
||||
@@ -77,6 +78,10 @@ fn request_with_opaque_headers(use_websocket: bool) -> CodexResponsesRequest {
|
||||
"x-openai-internal-codex-residency",
|
||||
STANDARD.encode(b"future-region")
|
||||
],
|
||||
[
|
||||
"X-OpenAI-Account-Routing-Override",
|
||||
STANDARD.encode(b"client-route")
|
||||
],
|
||||
["x-codex-turn-state", STANDARD.encode(b"turn-ascii")],
|
||||
["x-codex-turn-state", STANDARD.encode(b"turn-\x80")],
|
||||
["bad header name", STANDARD.encode(b"ignored")],
|
||||
@@ -202,7 +207,10 @@ async fn backend_http_should_preserve_business_headers_without_downstream_transp
|
||||
});
|
||||
// 不经 API 解码，直接构造协议上下文，验证 Provider 自身的过滤边界
|
||||
let request = request_with_opaque_headers(false);
|
||||
let profile = test_wire_profile();
|
||||
let mut configured_profile = test_wire_profile().snapshot();
|
||||
configured_profile.account_routing_override = Some("configured-route".into());
|
||||
let profile =
|
||||
provider_openai::transport::profile::CodexWireProfileState::new(configured_profile);
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_core_version = profile_snapshot.codex_version;
|
||||
@@ -277,7 +285,18 @@ async fn backend_http_should_preserve_business_headers_without_downstream_transp
|
||||
] {
|
||||
assert_eq!(raw_header_values(&raw, name), vec![value.to_vec()]);
|
||||
}
|
||||
assert!(raw_header_values(&raw, "traceparent").is_empty());
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "traceparent"),
|
||||
vec![b"synthetic-trace".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "x-openai-account-routing-override"),
|
||||
vec![b"configured-route".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "x-codex-installation-id"),
|
||||
vec![b"lease-installation".to_vec()]
|
||||
);
|
||||
let session = raw_header_values(&raw, "session-id");
|
||||
assert!(uuid::Uuid::parse_str(std::str::from_utf8(&session[0]).unwrap()).is_ok());
|
||||
// 指纹头由运行时画像生成，不透传客户端值
|
||||
@@ -295,7 +314,6 @@ async fn backend_http_should_preserve_business_headers_without_downstream_transp
|
||||
);
|
||||
for dropped in [
|
||||
"openai-beta",
|
||||
"x-codex-installation-id",
|
||||
"x-openai-actor-authorization",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
@@ -358,13 +376,15 @@ async fn backend_websocket_should_preserve_business_headers_without_downstream_t
|
||||
.expect("send opaque terminal event");
|
||||
});
|
||||
let request = request_with_opaque_headers(true);
|
||||
let mut profile = test_wire_profile().snapshot();
|
||||
profile.account_routing_override = Some("configured-route".into());
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("HTTP client"),
|
||||
format!("http://{address}"),
|
||||
test_wire_profile(),
|
||||
provider_openai::transport::profile::CodexWireProfileState::new(profile),
|
||||
)
|
||||
.with_websocket_pool(Arc::new(CodexWebSocketPool::new(Duration::from_mins(1))));
|
||||
|
||||
@@ -430,7 +450,6 @@ async fn backend_websocket_should_preserve_business_headers_without_downstream_t
|
||||
assert_eq!(values("version"), vec![b"1.2.3".to_vec()]);
|
||||
assert!(values("x-openai-internal-codex-residency").is_empty());
|
||||
for dropped in [
|
||||
"x-codex-installation-id",
|
||||
"x-openai-actor-authorization",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
@@ -462,7 +481,15 @@ async fn backend_websocket_should_preserve_business_headers_without_downstream_t
|
||||
] {
|
||||
assert_eq!(values(name), vec![value.to_vec()], "missing {name}");
|
||||
}
|
||||
assert!(values("traceparent").is_empty());
|
||||
assert_eq!(values("traceparent"), vec![b"synthetic-trace".to_vec()]);
|
||||
assert_eq!(
|
||||
values("x-openai-account-routing-override"),
|
||||
vec![b"configured-route".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("x-codex-installation-id"),
|
||||
vec![b"lease-installation".to_vec()]
|
||||
);
|
||||
let session = values("session-id");
|
||||
assert!(uuid::Uuid::parse_str(std::str::from_utf8(&session[0]).unwrap()).is_ok());
|
||||
assert_eq!(values("x-codex-turn-state"), vec![b"turn-ascii".to_vec()]);
|
||||
|
||||
@@ -410,42 +410,43 @@ async fn websocket_additional_rate_limit_resolves_by_name_without_touching_core(
|
||||
#[tokio::test]
|
||||
async fn quota_refresh_synchronizes_plan_changes_without_losing_subtypes() {
|
||||
for (current_plan, observed_plan, expected) in [
|
||||
("plus", Some("pro"), "pro"),
|
||||
("pro", Some("plus"), "plus"),
|
||||
("plus", Some("free"), "free"),
|
||||
("unknown", Some("pro"), "pro"),
|
||||
("plus", Some(" PRO "), "pro"),
|
||||
("pro", None, "pro"),
|
||||
("pro", Some("unknown"), "pro"),
|
||||
("pro", Some(" "), "pro"),
|
||||
(None, Some("free"), "free"),
|
||||
(Some("plus"), Some("pro"), "pro"),
|
||||
(Some("pro"), Some("plus"), "plus"),
|
||||
(Some("plus"), Some("free"), "free"),
|
||||
(Some("unknown"), Some("pro"), "pro"),
|
||||
(Some("plus"), Some(" PRO "), "pro"),
|
||||
(Some("pro"), None, "pro"),
|
||||
(Some("pro"), Some("unknown"), "pro"),
|
||||
(Some("pro"), Some(" "), "pro"),
|
||||
(
|
||||
"self_serve_business_prolite",
|
||||
Some("self_serve_business_prolite"),
|
||||
Some("team"),
|
||||
"self_serve_business_prolite",
|
||||
),
|
||||
(
|
||||
"self_serve_business_usage_based",
|
||||
Some("self_serve_business_usage_based"),
|
||||
Some("team"),
|
||||
"self_serve_business_usage_based",
|
||||
),
|
||||
(
|
||||
"enterprise_cbp_usage_based",
|
||||
Some("enterprise_cbp_usage_based"),
|
||||
Some("business"),
|
||||
"enterprise_cbp_usage_based",
|
||||
),
|
||||
("edu_plus", Some("edu"), "edu_plus"),
|
||||
("self_serve_business_prolite", Some("pro"), "pro"),
|
||||
(Some("edu_plus"), Some("edu"), "edu_plus"),
|
||||
(Some("self_serve_business_prolite"), Some("pro"), "pro"),
|
||||
(
|
||||
"team",
|
||||
Some("team"),
|
||||
Some("self_serve_business_prolite"),
|
||||
"self_serve_business_prolite",
|
||||
),
|
||||
("plus", Some("future_plan"), "future_plan"),
|
||||
(Some("plus"), Some("future_plan"), "future_plan"),
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_plan_refresh";
|
||||
let mut verified_account = profile("chatgpt-plan-refresh");
|
||||
verified_account.plan_type = Some(current_plan.to_owned());
|
||||
verified_account.plan_type = current_plan.map(str::to_owned);
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: account_id.to_owned(),
|
||||
@@ -480,7 +481,7 @@ async fn quota_refresh_synchronizes_plan_changes_without_losing_subtypes() {
|
||||
assert_eq!(
|
||||
after.plan_type(),
|
||||
Some(expected),
|
||||
"{current_plan} -> {observed_plan:?}"
|
||||
"{current_plan:?} -> {observed_plan:?}"
|
||||
);
|
||||
assert_eq!(after.revision(), before.revision());
|
||||
assert_eq!(after.email(), before.email());
|
||||
|
||||
@@ -71,12 +71,15 @@ fn assert_isolated_request(
|
||||
body: &Value,
|
||||
owner: Option<&str>,
|
||||
) {
|
||||
for name in ["x-openai-account-routing-override", "x-openai-fedramp"] {
|
||||
assert!(
|
||||
!headers.contains_key(name),
|
||||
"leaked {name} with owner {owner:?}"
|
||||
);
|
||||
}
|
||||
assert!(!headers.contains_key("x-openai-account-routing-override"));
|
||||
assert_eq!(
|
||||
headers
|
||||
.get_all("x-openai-fedramp")
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
["true", "false"]
|
||||
);
|
||||
assert_eq!(headers["x-codex-guardian"], "true");
|
||||
assert_eq!(
|
||||
headers
|
||||
@@ -112,7 +115,7 @@ fn assert_isolated_request(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_isolates_account_headers_and_only_scopes_guardian_parent_reference() {
|
||||
async fn http_applies_deployment_routing_and_only_scopes_guardian_parent_reference() {
|
||||
for owner in [Some(SELECTED_ACCOUNT), Some("acct_scope_old"), None] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
@@ -155,7 +158,7 @@ async fn http_isolates_account_headers_and_only_scopes_guardian_parent_reference
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_isolates_account_headers_and_only_scopes_guardian_parent_reference() {
|
||||
async fn websocket_applies_deployment_routing_and_only_scopes_guardian_parent_reference() {
|
||||
for owner in [Some(SELECTED_ACCOUNT), Some("acct_scope_old"), None] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
|
||||
@@ -136,15 +136,35 @@ async fn account_correlation_preserves_turns_across_transports_and_account_switc
|
||||
event.unwrap();
|
||||
}
|
||||
let (headers, body) = capture.await;
|
||||
assert!(!headers.contains_key("traceparent"));
|
||||
assert!(!headers.contains_key("tracestate"));
|
||||
assert!(!headers.contains_key("sentry-trace"));
|
||||
assert!(!headers.contains_key("baggage"));
|
||||
assert_eq!(
|
||||
headers["traceparent"],
|
||||
"00-11111111111111111111111111111111-2222222222222222-01"
|
||||
);
|
||||
assert_eq!(headers["tracestate"], "vendor=source");
|
||||
assert_eq!(headers["sentry-trace"], "source-trace");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get_all("baggage")
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
["sentry-environment=synthetic", "sentry-release=synthetic"]
|
||||
);
|
||||
let flat = &body["client_metadata"];
|
||||
assert!(flat.get("ws_request_header_traceparent").is_none());
|
||||
assert!(flat.get("ws_request_header_tracestate").is_none());
|
||||
assert!(flat.get("ws_request_header_sentry-trace").is_none());
|
||||
assert!(flat.get("ws_request_header_baggage").is_none());
|
||||
assert_eq!(
|
||||
flat["ws_request_header_traceparent"],
|
||||
headers["traceparent"].to_str().unwrap()
|
||||
);
|
||||
assert_eq!(flat["ws_request_header_tracestate"], "vendor=source");
|
||||
assert_eq!(flat["ws_request_header_sentry-trace"], "source-trace");
|
||||
assert_eq!(
|
||||
flat["ws_request_header_baggage"],
|
||||
"sentry-environment=synthetic"
|
||||
);
|
||||
assert_eq!(
|
||||
headers["x-codex-installation-id"].to_str().unwrap(),
|
||||
flat["x-codex-installation-id"]
|
||||
);
|
||||
let header: Value =
|
||||
serde_json::from_str(headers["x-codex-turn-metadata"].to_str().unwrap()).unwrap();
|
||||
let full: Value =
|
||||
@@ -163,8 +183,8 @@ async fn account_correlation_preserves_turns_across_transports_and_account_switc
|
||||
assert_eq!(flat["x-client-turn-id"], "source-turn");
|
||||
assert_eq!(flat["turn_id"], "source-turn");
|
||||
for metadata in [&header, &full] {
|
||||
assert!(metadata.get("sentry-trace").is_none());
|
||||
assert!(metadata.get("baggage").is_none());
|
||||
assert_eq!(metadata["sentry-trace"], "source-trace");
|
||||
assert_eq!(metadata["baggage"], "sentry-environment=synthetic");
|
||||
assert_eq!(metadata["turn_id"], "source-turn");
|
||||
assert_eq!(metadata["parent_turn_id"], "source-turn");
|
||||
assert_eq!(metadata["root_turn_id"], "source-turn");
|
||||
|
||||
@@ -20,6 +20,7 @@ mod search_model_access;
|
||||
mod selection_retry;
|
||||
mod session_binding;
|
||||
mod timing;
|
||||
mod turn_state;
|
||||
mod upstream_adapter;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
@@ -6009,7 +6010,7 @@ async fn opaque_provider_options_do_not_change_openai_account_selection() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_account_scoping_drops_downstream_installation_header() {
|
||||
async fn http_account_scoping_replaces_downstream_installation_header() {
|
||||
for owner in ["acct_scope_same", "acct_scope_old"] {
|
||||
let raw = r#"{"installation_id":"client-installation","future":true}"#;
|
||||
let request = capture_scoped_http_request(
|
||||
@@ -6050,7 +6051,10 @@ async fn http_account_scoping_drops_downstream_installation_header() {
|
||||
assert_ne!(installation_id, "client-installation");
|
||||
assert!(uuid::Uuid::parse_str(installation_id).is_ok());
|
||||
assert_eq!(body["installation_id"], installation_id);
|
||||
assert!(captured_header_values(&request, "x-codex-installation-id").is_empty());
|
||||
assert_eq!(
|
||||
captured_header_values(&request, "x-codex-installation-id"),
|
||||
vec![installation_id.as_bytes().to_vec()]
|
||||
);
|
||||
let headers = captured_header_values(&request, "x-codex-turn-metadata");
|
||||
assert_eq!(headers.len(), 1);
|
||||
for encoded in [
|
||||
@@ -6068,7 +6072,7 @@ async fn http_account_scoping_drops_downstream_installation_header() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_account_scoping_drops_downstream_installation_header() {
|
||||
async fn websocket_account_scoping_replaces_downstream_installation_header() {
|
||||
for owner in ["acct_scope_same", "acct_scope_old"] {
|
||||
let raw = r#"{"installation_id":"client-installation","future":true}"#;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -6147,7 +6151,7 @@ async fn websocket_account_scoping_drops_downstream_installation_header() {
|
||||
assert_ne!(installation_id, "client-installation");
|
||||
assert!(uuid::Uuid::parse_str(installation_id).is_ok());
|
||||
assert_eq!(body["installation_id"], installation_id);
|
||||
assert!(!header.contains_key("x-codex-installation-id"));
|
||||
assert_eq!(header["x-codex-installation-id"], installation_id);
|
||||
for encoded in [
|
||||
header["x-codex-turn-metadata"]
|
||||
.to_str()
|
||||
@@ -6219,7 +6223,10 @@ async fn same_account_scope_preserves_future_protocol_shapes() {
|
||||
body.pointer("/client_metadata/x-codex-installation-id")
|
||||
);
|
||||
assert!(body.pointer("/client_metadata/installation_id").is_none());
|
||||
assert!(captured_header_values(&request, "x-codex-installation-id").is_empty());
|
||||
assert_eq!(
|
||||
request.headers["x-codex-installation-id"].to_str().unwrap(),
|
||||
body["client_metadata"]["x-codex-installation-id"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -11528,6 +11535,88 @@ async fn api_key_websocket_uses_api_path_and_bearer_without_oauth_identity() {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_websocket_search_activation_reuses_original_connection() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_api_key(
|
||||
"acct_provider_contract",
|
||||
base,
|
||||
provider_openai::credential::ResponsesTransport::PreferWebsocket,
|
||||
)
|
||||
.await;
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = crate::transport::accept_codex_test_websocket(stream).await;
|
||||
for search in [false, true] {
|
||||
let frame = websocket.next().await.unwrap().unwrap();
|
||||
let payload: Value = serde_json::from_str(frame.to_text().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
payload["tools"].as_array().unwrap().len(),
|
||||
usize::from(search)
|
||||
);
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
crate::transport::completed_websocket_response("resp_api_search", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
let provider = provider(&store);
|
||||
let session_state = ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("account_id".into(), json!("acct_provider_contract")),
|
||||
("conversation_id".into(), json!("api-search-session")),
|
||||
("continuation_scope".into(), json!("connection_local")),
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
for tools in [json!([]), json!([{"type":"web_search"}])] {
|
||||
let operation = Operation::Generate(
|
||||
GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
json!({
|
||||
"model":"gpt-5.4", "input":"hello", "store":false,
|
||||
"session_id":"api-search-session", "tools":tools
|
||||
})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([
|
||||
("use_websocket".into(), json!(true)),
|
||||
(
|
||||
"downstream_websocket_connection_id".into(),
|
||||
json!("api-search-lane"),
|
||||
),
|
||||
])),
|
||||
)
|
||||
.with_provider_session_state(session_state.clone()),
|
||||
);
|
||||
timeout(Duration::from_secs(3), async {
|
||||
let mut stream = Arc::clone(&provider)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
diagnostic_context("req_api_search", "acct_provider_contract"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("API Key search activation should keep the same socket");
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
fn quota_continuation_operation(use_websocket: bool) -> Operation {
|
||||
Operation::Generate(
|
||||
GenerateRequest::from_protocol_payload(
|
||||
|
||||
@@ -453,6 +453,133 @@ async fn missing_catalog_unknown_plan_and_rotated_api_credentials_preserve_upstr
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_reasoning_value_preserves_account_eligibility_across_transports() {
|
||||
for transport in ["http", "sse", "websocket"] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (provider, pool) = provider_with_cooldowns(
|
||||
&store,
|
||||
format!("http://{}", listener.local_addr().unwrap()),
|
||||
cooldowns.clone(),
|
||||
);
|
||||
let upstream_error = json!({
|
||||
"code": "unsupported_value",
|
||||
"type": "invalid_request_error",
|
||||
"param": "reasoning.effort",
|
||||
"message": "Unsupported value: 'none' is not supported with the 'gpt-6-astra' model. Supported values are: 'low', 'medium', 'high', 'xhigh', and 'max'."
|
||||
});
|
||||
let expected_message = upstream_error["message"].as_str().unwrap().to_owned();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
if transport == "websocket" {
|
||||
let mut socket = accept_codex_test_websocket(socket).await;
|
||||
socket.next().await.unwrap().unwrap();
|
||||
socket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "error", "error": upstream_error, "status": 400
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
read_http_request(&mut socket).await;
|
||||
let (status, content_type, body) = if transport == "sse" {
|
||||
let event = json!({"type":"response.failed","response":{
|
||||
"id":"resp_bad_effort","status":"failed","output":[],"error":upstream_error
|
||||
}});
|
||||
(
|
||||
"200 OK",
|
||||
"text/event-stream",
|
||||
gateway_protocol::openai::sse::encode_sse_event(
|
||||
"response.failed",
|
||||
&event.to_string(),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"400 Bad Request",
|
||||
"application/json",
|
||||
json!({"error":upstream_error}).to_string(),
|
||||
)
|
||||
};
|
||||
socket.write_all(format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
).as_bytes()).await.unwrap();
|
||||
}
|
||||
// 合法请求仍应使用同一个账号，不能被上一请求的参数拒绝隔离
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
read_http_request(&mut socket).await;
|
||||
socket.write_all(format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
|
||||
CAPTURE_COMPLETED_SSE.len()
|
||||
).as_bytes()).await.unwrap();
|
||||
});
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
json!({
|
||||
"model":"gpt-6-astra","input":"hello","reasoning":{"effort":"none"}
|
||||
})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([(
|
||||
"use_websocket".to_owned(),
|
||||
json!(transport == "websocket"),
|
||||
)]));
|
||||
let mut stream = provider
|
||||
.clone()
|
||||
.execute(
|
||||
planned_request_for_model(
|
||||
"openai",
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload)),
|
||||
"gpt-6-astra",
|
||||
),
|
||||
context("req_unsupported_effort", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_, error) = super::precommit::stream_failure(&mut stream).await;
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
"{transport}"
|
||||
);
|
||||
assert!(error.retry_is_prohibited(), "{transport}");
|
||||
assert!(!error.replay_is_safe(), "{transport}");
|
||||
let visible = error.client_visible_upstream_error().unwrap();
|
||||
assert_eq!(visible.code(), Some("unsupported_value"));
|
||||
assert_eq!(visible.message(), expected_message);
|
||||
drop(stream);
|
||||
assert!(cooldowns.scoped.lock().unwrap().is_empty(), "{transport}");
|
||||
let mut recovered = provider
|
||||
.clone()
|
||||
.execute(
|
||||
planned_request_for_model("openai", operation("gpt-6-astra"), "gpt-6-astra"),
|
||||
context("req_valid_effort", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
recovered.metadata().provider_account_id().as_str(),
|
||||
"acct_provider_contract"
|
||||
);
|
||||
while let Some(event) = recovered.next().await {
|
||||
event.unwrap();
|
||||
}
|
||||
server.await.unwrap();
|
||||
pool.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upstream_model_rejection_cools_only_that_model_across_requests_and_transports() {
|
||||
use gateway_core::provider_ports::{ProviderCooldownScope, ProviderScopedCooldown};
|
||||
|
||||
@@ -85,6 +85,13 @@ async fn independent_endpoints_send_opaque_headers_without_client_identity() {
|
||||
let entries: Vec<_> = [
|
||||
("x-future-business", "first"),
|
||||
("x-future-business", "second"),
|
||||
("x-codex-installation-id", "client-installation"),
|
||||
("x-openai-account-routing-override", "client-route"),
|
||||
("x-openai-fedramp", "true"),
|
||||
("traceparent", "client-trace"),
|
||||
("tracestate", "vendor=client"),
|
||||
("sentry-trace", "sentry-client-trace"),
|
||||
("baggage", "environment=synthetic"),
|
||||
("authorization", "Bearer client-secret"),
|
||||
("chatgpt-account-id", "client-account"),
|
||||
("x-stainless-runtime", "client-runtime"),
|
||||
@@ -134,6 +141,21 @@ async fn independent_endpoints_send_opaque_headers_without_client_identity() {
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let sent = requests.last().unwrap();
|
||||
assert_eq!(sent.url.path(), endpoint);
|
||||
uuid::Uuid::parse_str(sent.headers["x-codex-installation-id"].to_str().unwrap()).unwrap();
|
||||
assert!(
|
||||
!sent
|
||||
.headers
|
||||
.contains_key("x-openai-account-routing-override")
|
||||
);
|
||||
for (name, expected) in [
|
||||
("x-openai-fedramp", "true"),
|
||||
("traceparent", "client-trace"),
|
||||
("tracestate", "vendor=client"),
|
||||
("sentry-trace", "sentry-client-trace"),
|
||||
("baggage", "environment=synthetic"),
|
||||
] {
|
||||
assert_eq!(sent.headers[name], expected);
|
||||
}
|
||||
if endpoint.ends_with("search") {
|
||||
assert_scoped_search_body(&sent.body, &raw);
|
||||
} else {
|
||||
|
||||
@@ -291,9 +291,9 @@ async fn privacy_header_change_isolates_websocket_then_reuses_stable_headers() {
|
||||
let mut sockets = Vec::new();
|
||||
let mut cache_keys = Vec::new();
|
||||
let mut session_headers = Vec::new();
|
||||
for (expected, requests) in [
|
||||
("Bearer first-synthetic-token", 1),
|
||||
("Bearer second-synthetic-token", 2),
|
||||
for (expected, trace, requests) in [
|
||||
("Bearer first-synthetic-token", "first-trace", 1),
|
||||
("Bearer second-synthetic-token", "second-trace", 2),
|
||||
] {
|
||||
let (stream, _) = timeout(Duration::from_secs(5), listener.accept())
|
||||
.await
|
||||
@@ -302,8 +302,11 @@ async fn privacy_header_change_isolates_websocket_then_reuses_stable_headers() {
|
||||
let mut socket =
|
||||
crate::transport::accept_codex_test_websocket_with(stream, |request, response| {
|
||||
assert_eq!(request.headers()["authorization"], expected);
|
||||
assert!(!request.headers().contains_key("sentry-trace"));
|
||||
assert!(!request.headers().contains_key("baggage"));
|
||||
assert_eq!(request.headers()["sentry-trace"], trace);
|
||||
assert_eq!(
|
||||
request.headers()["baggage"],
|
||||
format!("sentry-trace_id={trace}")
|
||||
);
|
||||
session_headers.push(request.headers()["session-id"].clone());
|
||||
response.headers_mut().insert(
|
||||
"sec-websocket-extensions",
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
//! 验证同轮全量请求回传的 token 归属、失败路径与账号状态隔离
|
||||
|
||||
use super::*;
|
||||
|
||||
const TOKEN: &str = "upstream-turn-state";
|
||||
|
||||
fn full_body() -> Value {
|
||||
json!({
|
||||
"model": "gpt-5.4", "store": false,
|
||||
"input": [{"role": "user", "content": "hello"}],
|
||||
"client_metadata": {
|
||||
"session_id": "turn-state-session",
|
||||
"thread_id": "turn-state-thread",
|
||||
"turn_id": "turn-state-turn"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn full_request(body: Value, token: Option<&str>, websocket: bool) -> Operation {
|
||||
let mut protocol_context = Map::from_iter([
|
||||
("use_websocket".to_owned(), json!(websocket)),
|
||||
(
|
||||
"session_id".to_owned(),
|
||||
body["client_metadata"]["session_id"].clone(),
|
||||
),
|
||||
(
|
||||
"thread_id".to_owned(),
|
||||
body["client_metadata"]["thread_id"].clone(),
|
||||
),
|
||||
(
|
||||
"turn_metadata".to_owned(),
|
||||
json!(body["client_metadata"].to_string()),
|
||||
),
|
||||
]);
|
||||
if let Some(token) = token {
|
||||
protocol_context.insert("turn_state".to_owned(), json!(token));
|
||||
protocol_context.insert(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([["x-codex-turn-state", STANDARD.encode(token)]]),
|
||||
);
|
||||
}
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body.as_object().unwrap().clone())
|
||||
.unwrap()
|
||||
.with_context(protocol_context),
|
||||
))
|
||||
}
|
||||
|
||||
async fn collect_response(
|
||||
provider: &Arc<CodexProvider>,
|
||||
operation: Operation,
|
||||
context: AttemptContext,
|
||||
) -> Result<bool, gateway_core::error::ProviderError> {
|
||||
let mut stream = provider
|
||||
.clone()
|
||||
.execute(planned_request("openai", operation), context)
|
||||
.await?;
|
||||
let mut observed_token = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event?;
|
||||
if let Some(observation) = event.response_observation() {
|
||||
observed_token |= observation.client_headers().iter().any(|header| {
|
||||
header.name() == "x-codex-turn-state" && header.value().as_ref() == TOKEN.as_bytes()
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(observed_token)
|
||||
}
|
||||
|
||||
async fn http_fixture() -> (Arc<MemoryAccountStore>, MockServer, Arc<CodexProvider>) {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_session_affinity").await;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.insert_header("x-codex-turn-state", TOKEN)
|
||||
.set_body_string(CAPTURE_COMPLETED_SSE),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let provider = provider_with_base_url(&store, server.uri());
|
||||
(store, server, provider)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_turn_full_http_request_preserves_observed_token_without_restoring_other_account_state()
|
||||
{
|
||||
let (_store, server, provider) = http_fixture().await;
|
||||
assert!(
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, false),
|
||||
context("req_turn_first", CancellationToken::new())
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let mut body = full_body();
|
||||
body["client_metadata"]["x-codex-turn-state"] = json!(TOKEN);
|
||||
body["client_metadata"]["response_id"] = json!("unowned-response");
|
||||
body["client_metadata"]["parent_response_id"] = json!("unowned-parent");
|
||||
body["client_metadata"]["account_id"] = json!("unowned-account");
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(body, Some(TOKEN), false),
|
||||
context("req_turn_echo", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[1].headers["x-codex-turn-state"], TOKEN);
|
||||
let body = captured_request_body(&requests[1]);
|
||||
assert_eq!(body["client_metadata"]["x-codex-turn-state"], TOKEN);
|
||||
assert!(body.get("previous_response_id").is_none());
|
||||
for field in ["response_id", "parent_response_id", "account_id"] {
|
||||
assert!(
|
||||
body["client_metadata"].get(field).is_none(),
|
||||
"{field} must remain scoped"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_header_echo_does_not_authorize_a_different_metadata_token() {
|
||||
let (_store, server, provider) = http_fixture().await;
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, false),
|
||||
context("req_turn_first", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut body = full_body();
|
||||
body["client_metadata"]["x-codex-turn-state"] = json!("unobserved-token");
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(body, Some(TOKEN), false),
|
||||
context("req_turn_echo", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests[1].headers["x-codex-turn-state"], TOKEN);
|
||||
assert!(
|
||||
captured_request_body(&requests[1])["client_metadata"]
|
||||
.get("x-codex-turn-state")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_state_from_failed_http_response_is_owned_before_a_terminal_success() {
|
||||
let (_store, server, provider) = http_fixture().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(ResponseTemplate::new(400)
|
||||
.insert_header("x-codex-turn-state", TOKEN)
|
||||
.set_body_json(json!({"error": {"type": "invalid_request_error", "message": "Invalid request."}})))
|
||||
.up_to_n_times(1)
|
||||
.with_priority(1)
|
||||
.mount(&server).await;
|
||||
assert!(
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, false),
|
||||
context("req_turn_first", CancellationToken::new())
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), Some(TOKEN), false),
|
||||
context("req_turn_echo", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[1].headers["x-codex-turn-state"], TOKEN);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_state_proof_does_not_authorize_an_unowned_previous_response_id() {
|
||||
let (_store, server, provider) = http_fixture().await;
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, false),
|
||||
context("req_turn_first", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut body = full_body();
|
||||
body["previous_response_id"] = json!("unowned-response");
|
||||
let error = collect_response(
|
||||
&provider,
|
||||
full_request(body, Some(TOKEN), false),
|
||||
context("req_turn_echo", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_error()
|
||||
.and_then(|detail| detail.code()),
|
||||
Some("previous_response_not_found")
|
||||
);
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_request_turn_state_requires_matching_client_account_thread_turn_and_token() {
|
||||
for case in [
|
||||
"new_turn",
|
||||
"missing_turn",
|
||||
"new_thread",
|
||||
"new_session",
|
||||
"new_key",
|
||||
"new_account",
|
||||
"unknown_token",
|
||||
"conflicting_owner",
|
||||
"expired",
|
||||
"new_provider",
|
||||
] {
|
||||
let (store, server, provider) = http_fixture().await;
|
||||
let first_context = if case == "expired" {
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_turn_first").unwrap(),
|
||||
ClientApiKeyId::new("key_openai_contract").unwrap(),
|
||||
),
|
||||
NonZeroU32::new(1).unwrap(),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy().with_openai_session_affinity_ttl(Duration::ZERO),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, None)
|
||||
.with_account_scope(contract_account_scope()),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
} else if case == "new_account" {
|
||||
diagnostic_context("req_turn_first", "acct_session_affinity")
|
||||
} else {
|
||||
context("req_turn_first", CancellationToken::new())
|
||||
};
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, false),
|
||||
first_context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut body = full_body();
|
||||
let mut token = TOKEN;
|
||||
let mut echo_context = context("req_turn_echo", CancellationToken::new());
|
||||
let provider = if case == "new_provider" {
|
||||
provider_with_base_url(&store, server.uri())
|
||||
} else {
|
||||
provider
|
||||
};
|
||||
match case {
|
||||
"new_turn" => body["client_metadata"]["turn_id"] = json!("another-turn"),
|
||||
"missing_turn" => {
|
||||
body["client_metadata"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("turn_id");
|
||||
}
|
||||
"new_thread" => body["client_metadata"]["thread_id"] = json!("another-thread"),
|
||||
"new_session" => {
|
||||
body["client_metadata"]["session_id"] = json!("another-session");
|
||||
echo_context = diagnostic_context("req_turn_echo", "acct_session_affinity");
|
||||
}
|
||||
"unknown_token" => token = "unobserved-token",
|
||||
"new_account" => {
|
||||
create_account(&store, "acct_scope_new").await;
|
||||
echo_context = diagnostic_context("req_turn_echo", "acct_scope_new");
|
||||
}
|
||||
"conflicting_owner" => {
|
||||
echo_context = context_with_state_owner("req_turn_echo", "acct_scope_old")
|
||||
}
|
||||
"new_key" => {
|
||||
echo_context = AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_turn_echo").unwrap(),
|
||||
ClientApiKeyId::new("another-key").unwrap(),
|
||||
),
|
||||
NonZeroU32::new(1).unwrap(),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::diagnostic(
|
||||
BTreeSet::new(),
|
||||
ProviderAccountId::new("acct_session_affinity").unwrap(),
|
||||
None,
|
||||
),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
body["client_metadata"]["x-codex-turn-state"] = json!(token);
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(body, Some(token), false),
|
||||
echo_context,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{case}: {error:?}"));
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 2, "{case}");
|
||||
assert!(
|
||||
!requests[1].headers.contains_key("x-codex-turn-state"),
|
||||
"{case}"
|
||||
);
|
||||
assert!(
|
||||
captured_request_body(&requests[1])["client_metadata"]
|
||||
.get("x-codex-turn-state")
|
||||
.is_none(),
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_metadata_turn_state_survives_full_replay_after_completion_or_failure() {
|
||||
for fail_first in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_session_affinity").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.metadata", "headers": {"x-codex-turn-state": [TOKEN]}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
if fail_first {
|
||||
websocket.close(None).await.unwrap();
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
websocket = accept_codex_test_websocket(stream).await;
|
||||
} else {
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
CAPTURE_COMPLETED_SSE
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("data: "))
|
||||
.unwrap()
|
||||
.to_owned()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let echo: Value = loop {
|
||||
match websocket.next().await.unwrap().unwrap() {
|
||||
Message::Text(text) => break serde_json::from_str(&text).unwrap(),
|
||||
Message::Ping(payload) => websocket.send(Message::Pong(payload)).await.unwrap(),
|
||||
Message::Close(_) => {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
websocket = accept_codex_test_websocket(stream).await;
|
||||
}
|
||||
other => panic!("expected response.create, got {other:?}"),
|
||||
}
|
||||
};
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
CAPTURE_COMPLETED_SSE
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("data: "))
|
||||
.unwrap()
|
||||
.to_owned()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
echo
|
||||
});
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let first = collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), None, true),
|
||||
context("req_turn_first", CancellationToken::new()),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(first.is_err(), fail_first);
|
||||
collect_response(
|
||||
&provider,
|
||||
full_request(full_body(), Some(TOKEN), true),
|
||||
context("req_turn_echo", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let echo = timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
echo["client_metadata"]["x-codex-turn-state"], TOKEN,
|
||||
"failed first response: {fail_first}"
|
||||
);
|
||||
assert!(echo.get("previous_response_id").is_none());
|
||||
}
|
||||
}
|
||||
@@ -428,6 +428,10 @@ fn live_protocol_headers() -> Vec<gateway_core::operation::ProviderHttpHeader> {
|
||||
("traceparent", "client-trace"),
|
||||
("tracestate", "client-state"),
|
||||
("ws_request_header_traceparent", "client-ws-trace"),
|
||||
("x-codex-installation-id", "foreign-installation"),
|
||||
("X-OpenAI-Account-Routing-Override", "first-route"),
|
||||
("x-openai-account-routing-override", "second-route"),
|
||||
("x-openai-fedramp", "true"),
|
||||
("x-codex-turn-state", "foreign-turn-state"),
|
||||
("x-codex-routing-hint", "foreign-route"),
|
||||
("openai-alpha", "live-test"),
|
||||
@@ -452,6 +456,7 @@ fn assert_live_correlation(headers: &reqwest::header::HeaderMap) {
|
||||
uuid::Uuid::parse_str(session).unwrap();
|
||||
assert_eq!(headers["thread-id"], session);
|
||||
assert_eq!(headers["openai-alpha"], "live-test");
|
||||
assert_eq!(headers["x-openai-fedramp"], "true");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get_all("x-test-business")
|
||||
@@ -465,9 +470,7 @@ fn assert_live_correlation(headers: &reqwest::header::HeaderMap) {
|
||||
"x-oai-attestation",
|
||||
"x-forwarded-for",
|
||||
"x-stainless-runtime",
|
||||
"traceparent",
|
||||
"tracestate",
|
||||
"ws_request_header_traceparent",
|
||||
"x-openai-account-routing-override",
|
||||
"x-codex-turn-state",
|
||||
"x-codex-routing-hint",
|
||||
] {
|
||||
@@ -479,7 +482,14 @@ fn assert_live_correlation(headers: &reqwest::header::HeaderMap) {
|
||||
assert_eq!(metadata["future"], true);
|
||||
assert_ne!(metadata["installation_id"], "foreign-installation");
|
||||
assert!(metadata.get("account_id").is_none());
|
||||
assert!(metadata.get("traceparent").is_none());
|
||||
assert_eq!(metadata["traceparent"], "client-trace");
|
||||
assert_eq!(headers["traceparent"], "client-trace");
|
||||
assert_eq!(headers["tracestate"], "client-state");
|
||||
assert_eq!(headers["ws_request_header_traceparent"], "client-ws-trace");
|
||||
assert_eq!(
|
||||
headers["x-codex-installation-id"].to_str().unwrap(),
|
||||
metadata["installation_id"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -22,6 +22,59 @@ fn classify(status: u16, body: &str) -> CodexFailureCategory {
|
||||
.category()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_rejection_requires_the_official_code_and_status() {
|
||||
for status in [400, 403, 404] {
|
||||
assert_eq!(
|
||||
classify(
|
||||
status,
|
||||
r#"{"error":{"code":"model_not_found","message":"Access to this deployment ended."}}"#
|
||||
),
|
||||
CodexFailureCategory::ModelUnsupported
|
||||
);
|
||||
}
|
||||
for (status, body, expected) in [
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"code":"unsupported_value","type":"invalid_request_error","param":"reasoning.effort","message":"Unsupported value: 'none' is not supported with the 'gpt-6-astra' model. Supported values are: 'low', 'medium', 'high', 'xhigh', and 'max'."}}"#,
|
||||
CodexFailureCategory::InvalidRequest,
|
||||
),
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"message":"The model is not supported."}}"#,
|
||||
CodexFailureCategory::InvalidRequest,
|
||||
),
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"code":"unknown_error","message":"model_not_supported"}}"#,
|
||||
CodexFailureCategory::InvalidRequest,
|
||||
),
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"code":"unknown_error"},"detail":"model_not_available"}"#,
|
||||
CodexFailureCategory::InvalidRequest,
|
||||
),
|
||||
(
|
||||
500,
|
||||
r#"{"error":{"code":"model_not_found","message":"The model is not available."}}"#,
|
||||
CodexFailureCategory::Unavailable,
|
||||
),
|
||||
] {
|
||||
let failure = CodexUpstreamFailure::from_response(
|
||||
reqwest::StatusCode::from_u16(status).unwrap(),
|
||||
body,
|
||||
None,
|
||||
&CodexUpstreamDiagnostics::default(),
|
||||
None,
|
||||
&[],
|
||||
&[],
|
||||
CodexUpstreamSendPhase::AfterPayload,
|
||||
);
|
||||
assert_eq!(failure.category(), expected, "{body}");
|
||||
assert!(!failure.replay_is_safe(), "{body}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flex_capacity_rejection_should_be_terminal_even_before_payload_send() {
|
||||
for phase in [
|
||||
|
||||
@@ -226,6 +226,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
("x-codex-parent-thread-id", "parent-456"),
|
||||
("x-openai-subagent", "future_codex_mode"),
|
||||
("x-openai-memgen-request", "true"),
|
||||
("x-codex-installation-id", "install-123"),
|
||||
("session-id", "cp_derived"),
|
||||
] {
|
||||
let value = &headers.iter().find(|(header, _)| header == name).unwrap().1;
|
||||
@@ -240,7 +241,6 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
}
|
||||
}
|
||||
for forbidden in [
|
||||
"x-codex-installation-id",
|
||||
"x-codex-turn-id",
|
||||
"x-openai-internal-codex-residency",
|
||||
"accept",
|
||||
@@ -365,8 +365,11 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
] {
|
||||
assert!(header_names.iter().any(|name| name == required));
|
||||
}
|
||||
assert_eq!(
|
||||
read_header_value(&raw_request, "x-codex-installation-id"),
|
||||
Some("install-1")
|
||||
);
|
||||
for forbidden in [
|
||||
"x-codex-installation-id",
|
||||
"x-codex-turn-id",
|
||||
"x-openai-internal-codex-residency",
|
||||
"sec-ch-ua",
|
||||
|
||||
@@ -260,6 +260,7 @@ async fn slow_pooled_openings_should_remain_healthy_and_serve_subsequent_request
|
||||
let payload = websocket.next().await.unwrap().unwrap();
|
||||
let payload: Value = serde_json::from_str(payload.to_text().unwrap()).unwrap();
|
||||
assert_eq!(payload["input"][0]["content"], "subsequent request");
|
||||
assert_eq!(payload["tools"][0]["type"], "web_search");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_background_ready", 2, 1).into(),
|
||||
@@ -300,11 +301,14 @@ async fn slow_pooled_openings_should_remain_healthy_and_serve_subsequent_request
|
||||
);
|
||||
websocket_ready_rx.recv().await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
let mut subsequent = codex_request(
|
||||
let mut body = codex_request_body(
|
||||
"gpt-5.5",
|
||||
"be brief",
|
||||
vec![json!({"role":"user","content":"subsequent request"})],
|
||||
);
|
||||
// 后台只完成握手，首次写入可以直接声明搜索，不应按无搜索首包回收
|
||||
body.insert("tools".into(), json!([{"type":"web_search"}]));
|
||||
let mut subsequent = CodexResponsesRequest::from_body(body);
|
||||
subsequent.use_websocket = true;
|
||||
subsequent.local_conversation_id = request.local_conversation_id.clone();
|
||||
let second = backend
|
||||
|
||||
@@ -143,7 +143,7 @@ fn deployment_routing_survives_artifact_updates_and_profile_selection() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_transports_apply_configured_routing_and_discard_downstream_values() {
|
||||
async fn model_transports_apply_deployment_routing_with_or_without_client_headers() {
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use gateway_core::operation::{GenerateRequest, ProtocolPayload};
|
||||
|
||||
@@ -178,7 +178,9 @@ async fn model_transports_apply_configured_routing_and_discard_downstream_values
|
||||
[None, Some("NO_CONSTRAINT")],
|
||||
),
|
||||
] {
|
||||
for websocket in [false, true] {
|
||||
for (websocket, has_downstream_headers) in
|
||||
[(false, false), (false, true), (true, false), (true, true)]
|
||||
{
|
||||
let (base_url, capture): (
|
||||
String,
|
||||
futures::future::BoxFuture<'static, reqwest::header::HeaderMap>,
|
||||
@@ -237,6 +239,7 @@ async fn model_transports_apply_configured_routing_and_discard_downstream_values
|
||||
);
|
||||
let downstream_headers = HEADERS
|
||||
.into_iter()
|
||||
.filter(|_| has_downstream_headers)
|
||||
.flat_map(|name| {
|
||||
["downstream-first", "downstream-second"]
|
||||
.map(|value| json!([name, STANDARD.encode(value)]))
|
||||
@@ -270,7 +273,7 @@ async fn model_transports_apply_configured_routing_and_discard_downstream_values
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
value.into_iter().collect::<Vec<_>>(),
|
||||
"{document}, websocket={websocket}, {name}",
|
||||
"{document}, websocket={websocket}, downstream={has_downstream_headers}, {name}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ mod cleanup;
|
||||
mod diagnostics;
|
||||
mod fingerprint;
|
||||
mod metadata;
|
||||
mod pool;
|
||||
|
||||
#[test]
|
||||
fn outbound_websocket_diagnostics_do_not_expose_target_credentials() {
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
//! 搜索能力变化时的连接复用与原生续接恢复
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn search_test_server() -> (
|
||||
CodexBackendClient,
|
||||
Arc<CodexWebSocketPool>,
|
||||
tokio::sync::mpsc::UnboundedReceiver<(usize, Value)>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut connection = 0;
|
||||
while let Ok((socket, _)) = listener.accept().await {
|
||||
connection += 1;
|
||||
let mut websocket = accept_codex_test_websocket(socket).await;
|
||||
while let Some(Ok(message)) = websocket.next().await {
|
||||
match message {
|
||||
Message::Text(text) => {
|
||||
let request: Value = serde_json::from_str(&text).unwrap();
|
||||
tx.send((connection, request)).unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_search", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
Message::Ping(_) => websocket.flush().await.unwrap(),
|
||||
Message::Close(_) => {
|
||||
websocket.flush().await.unwrap();
|
||||
break;
|
||||
}
|
||||
other => panic!("unexpected frame: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
(client, pool, rx, server)
|
||||
}
|
||||
|
||||
fn search_request(downstream_ws: bool, overrides: Value) -> CodexResponsesRequest {
|
||||
let mut body = codex_request_body("gpt-test", "be brief", vec![]);
|
||||
body.insert("store".into(), json!(false));
|
||||
body.extend(overrides.as_object().unwrap().clone());
|
||||
let mut request = CodexResponsesRequest::from_body(body);
|
||||
request.local_conversation_id = Some("search-session".into());
|
||||
request.downstream_websocket_connection_id = downstream_ws.then(|| "search-lane".into());
|
||||
request
|
||||
}
|
||||
|
||||
async fn send(
|
||||
client: &CodexBackendClient,
|
||||
request: &CodexResponsesRequest,
|
||||
) -> CodexClientResult<CollectedBackendResponse> {
|
||||
timeout(
|
||||
Duration::from_secs(3),
|
||||
client.create_response(
|
||||
request,
|
||||
request_context("req_search", Some("search-account")),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("request must finish without waiting for pool expiry")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_activation_reconnects_for_http_and_websocket_ingress() {
|
||||
for downstream_ws in [false, true] {
|
||||
for (warmup, mut declaration) in [
|
||||
(
|
||||
false,
|
||||
json!({"tools":[{"type":"web_search", "search_content_types":["text","image"]}]}),
|
||||
),
|
||||
(
|
||||
true,
|
||||
json!({"tools":[{"type":"web_search", "external_web_access":false}]}),
|
||||
),
|
||||
(
|
||||
false,
|
||||
json!({"input":[{"type":"additional_tools", "role":"developer", "tools":[{"type":"web_search"}]}]}),
|
||||
),
|
||||
(
|
||||
false,
|
||||
json!({"input":[{"type":"tool_search_output", "tools":[{"type":"web_search_preview"}]}]}),
|
||||
),
|
||||
] {
|
||||
let (client, pool, mut seen, server) = search_test_server().await;
|
||||
let first = search_request(
|
||||
downstream_ws,
|
||||
json!({
|
||||
"generate": !warmup,
|
||||
"tools": [{"type":"function", "name":"web_search", "parameters":{}}]
|
||||
}),
|
||||
);
|
||||
send(&client, &first).await.unwrap();
|
||||
assert_eq!(seen.recv().await.unwrap().0, 1);
|
||||
|
||||
declaration
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.entry("input")
|
||||
.or_insert_with(|| json!([]));
|
||||
declaration["input"].as_array_mut().unwrap().extend([
|
||||
json!({"type":"function_call", "name":"lookup", "call_id":"call_1", "arguments":"{}"}),
|
||||
json!({"type":"function_call_output", "call_id":"call_1", "output":"synthetic history"}),
|
||||
]);
|
||||
let second = search_request(downstream_ws, declaration);
|
||||
let response = send(&client, &second).await.unwrap();
|
||||
let (connection, payload) = seen.recv().await.unwrap();
|
||||
pool.shutdown().await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(connection, 2, "search activation must use a fresh socket");
|
||||
assert_eq!(response.websocket_pool_decision.unwrap().kind(), "new");
|
||||
assert_eq!(payload["input"], second.body()["input"]);
|
||||
assert_eq!(payload.get("tools"), second.body().get("tools"));
|
||||
assert!(payload.get("previous_response_id").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_initialized_socket_survives_tool_removal_and_image_search_activation() {
|
||||
let (client, pool, mut seen, server) = search_test_server().await;
|
||||
for tools in [
|
||||
json!([{"type":"web_search", "external_web_access":false}]),
|
||||
json!([]),
|
||||
json!([{"type":"web_search", "search_content_types":["text","image"]}]),
|
||||
] {
|
||||
let request = search_request(true, json!({"tools": tools}));
|
||||
send(&client, &request).await.unwrap();
|
||||
let (connection, payload) = seen.recv().await.unwrap();
|
||||
assert_eq!(connection, 1);
|
||||
assert_eq!(payload["tools"], tools);
|
||||
}
|
||||
pool.shutdown().await;
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_activation_rejects_native_delta_then_accepts_full_replay() {
|
||||
let (client, pool, mut seen, server) = search_test_server().await;
|
||||
let request = search_request(true, json!({}));
|
||||
send(&client, &request).await.unwrap();
|
||||
seen.recv().await.unwrap();
|
||||
let mut request = search_request(true, json!({"tools":[{"type":"web_search"}]}));
|
||||
request.set_previous_response_id(Some("resp_search".into()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let result = send(&client, &request).await;
|
||||
if result.is_ok() {
|
||||
pool.shutdown().await;
|
||||
server.abort();
|
||||
panic!("connection-local delta must require client replay before sending");
|
||||
}
|
||||
let CodexClientError::WebSocket(error) = result.unwrap_err() else {
|
||||
panic!("expected a typed continuation error");
|
||||
};
|
||||
assert!(error.continuation_unavailable_reason().is_some());
|
||||
assert_eq!(
|
||||
error.connection_observation().unwrap().exit_reason(),
|
||||
"search_activation_required"
|
||||
);
|
||||
assert!(seen.try_recv().is_err(), "delta must never reach upstream");
|
||||
|
||||
let request = search_request(
|
||||
true,
|
||||
json!({
|
||||
"tools":[{"type":"web_search"}],
|
||||
"input":[{"role":"user", "content":"complete synthetic history"}]
|
||||
}),
|
||||
);
|
||||
send(&client, &request).await.unwrap();
|
||||
let (connection, payload) = seen.recv().await.unwrap();
|
||||
assert_eq!(connection, 2);
|
||||
assert_eq!(payload["input"], request.body()["input"]);
|
||||
assert!(payload.get("previous_response_id").is_none());
|
||||
pool.shutdown().await;
|
||||
server.abort();
|
||||
}
|
||||
@@ -34,3 +34,4 @@ a0e409402611dbc76bb71393b8eda7c717756c18974dbabca305fdcb4b115f31 0032_usage_and
|
||||
9c15a6b6c1f9eab8ad478772f98f8fc4a997b825d58d12cffe1a15579c5785b3 0034_account_error_protection.sql
|
||||
bd73379456839e8a12d7591ad2a839901af0f5ba27871248569abdfa26bc83ef 0035_cpr_link_and_concurrency_defaults.sql
|
||||
d05442ee4cb85db8131e015dbed9c6ad988f93306abcda34647317674857ee44 0036_account_display_names.sql
|
||||
b215ea90d5c8753aeb8192b17ffe65f7a95c1d620aaeaf9fc4337d8cf6f2634e 0037_retry_after_wait_limit.sql
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- 网关接受的单次上游重试等待上限，零表示不接受正数等待
|
||||
ALTER TABLE runtime_settings
|
||||
ADD COLUMN max_retry_after_seconds BIGINT NOT NULL DEFAULT 240
|
||||
CHECK (max_retry_after_seconds BETWEEN 0 AND 4294967295);
|
||||
+3
-3
@@ -4,7 +4,7 @@
|
||||
# 版本严格一致。升级数据库版本时只需同步修改这里与 deploy/compose.yaml。
|
||||
ARG POSTGRES_IMAGE=postgres:18-bookworm@sha256:bbf83594798e144bff365a9c191c5d779f181fb3135902064ab8ff11918020fc
|
||||
|
||||
FROM node:26.12.1-bookworm-slim@sha256:28147155d47dc31abe0a46e0d12b59b2fb3d0a2dcc5dc65fce76a074180b16fe AS frontend-builder
|
||||
FROM node:26.13.0-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 AS frontend-builder
|
||||
|
||||
WORKDIR /app/frontend
|
||||
COPY frontend/package.json frontend/pnpm-lock.yaml frontend/pnpm-workspace.yaml ./
|
||||
@@ -13,7 +13,7 @@ RUN npm install --global "$(node -p 'require("./package.json").packageManager')"
|
||||
COPY frontend ./
|
||||
RUN pnpm build
|
||||
|
||||
FROM rust:1.99.1-bookworm@sha256:48b7f435b8fc6378ce43b44f15e3a47fd8919b2825f835c39bd4bd3da50af5b8 AS backend-builder
|
||||
FROM rust:1.99.1-bookworm@sha256:9181b56c2a21ab1d5e73cffe04fc762426d30aa7fac3c91141dd033f49a7fc8b AS backend-builder
|
||||
|
||||
WORKDIR /app/backend
|
||||
ARG TARGETARCH
|
||||
@@ -40,7 +40,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
# 备份客户端来源：与 compose 服务端同一固定镜像，版本一致。
|
||||
FROM ${POSTGRES_IMAGE} AS pg-client
|
||||
|
||||
FROM debian:bookworm-slim@sha256:7ff6ba31527b9e336162fa061746c366c4b2032f4d2cff2078bea43f78efe3f8 AS runtime-base
|
||||
FROM debian:bookworm-slim@sha256:99c0b25c46d0214d5a2091092e7908907dcaecd6a962cde0ac700acee222b6f3 AS runtime-base
|
||||
|
||||
ENV TZ=UTC
|
||||
|
||||
|
||||
+166
-6
@@ -6,7 +6,7 @@
|
||||
|
||||
| 任务 | 入口 |
|
||||
| --- | --- |
|
||||
| 安装与启动 | [手动安装](#手动安装) · [启动](#启动) · [公网访问](#公网访问) |
|
||||
| 安装与启动 | [手动安装](#手动安装) · [Windows 二进制部署](#windows-二进制部署) · [启动](#启动) · [公网访问](#公网访问) |
|
||||
| 接入客户端 | [客户端配置](#客户端配置) · [时区与客户端环境信息](#时区与客户端环境信息) · [登录与生图排查](#登录与生图排查) |
|
||||
| 日常运维 | [运行设置](#启动后的运行设置) · [小内存优化](#小内存优化) · [持久化与日志](#持久化与备份) · [请求排查](#请求错误排查) · [密码轮换](#密码语义) |
|
||||
| 更新与恢复 | [镜像升级](#镜像升级与源码构建) · [在线更新](#管理端在线更新) · [备份与恢复](#备份与恢复) · [停止与重启](#停止与重启) |
|
||||
@@ -19,7 +19,7 @@
|
||||
| 位置 | 内容 | 修改方式 |
|
||||
| --- | --- | --- |
|
||||
| `deploy/config.yaml` | 监听、数据库/Redis 连接、部署凭据、时区、日志与可选内存分配策略等启动配置 | 从模板创建，已有部署合并修改，不覆盖原文件 |
|
||||
| `deploy/compose.yaml` | 镜像、容器网络、端口、挂载与资源限制 | 调整 Compose 并重建受影响容器 |
|
||||
| `deploy/compose.yaml` | 镜像、容器网络、端口、挂载、PostgreSQL 启动参数与资源限制 | 调整 Compose 并重建受影响容器 |
|
||||
| PostgreSQL | 账号、Key、运行设置、上游身份、插件配置等业务数据 | 管理端或管理 API |
|
||||
| 浏览器本地 | 主题和界面偏好 | 当前浏览器的主题设置 |
|
||||
|
||||
@@ -58,6 +58,21 @@ host:
|
||||
|
||||
### 小内存优化
|
||||
|
||||
1 GB 内存、网关与 PostgreSQL / Redis 同机、低并发部署可从本节参数开始评估。
|
||||
这些值是调优起点，不是整机内存上限或容量保证；先记录实际负载下的内存峰值，优先缩小数据库连接池和缓冲，再按需调整 Redis 与应用空闲内存回收
|
||||
|
||||
| 调整对象 | 配置入口 | 生效方式 |
|
||||
| --- | --- | --- |
|
||||
| 应用空闲内存回收 | `config.yaml` → `services.app-runtime.environment.GLIBC_TUNABLES` | 重建应用容器 |
|
||||
| 应用 PostgreSQL 连接池 | `config.yaml` → `store.pool.max_connections` | 重启应用；Compose 可按下文重建 |
|
||||
| PostgreSQL 内存与连接预算 | `compose.yaml` → `services.postgres.command` | 重建 PostgreSQL 容器 |
|
||||
| Redis 持久化与内存预算 | `config.yaml` → `services.redis-credentials.command` | 重建 Redis 容器 |
|
||||
|
||||
以下 YAML 片段合并到已有节点，不要重复创建顶层 `store` / `services`，也不要覆盖凭据、挂载和健康检查。
|
||||
这些参数没有管理端热更新入口；二进制部署沿用 `store.pool`，PostgreSQL / Redis 参数则配置到各自服务的配置文件或启动命令，应用不会读取 `services` 桥接区来修改外部数据库
|
||||
|
||||
#### 应用空闲内存回收
|
||||
|
||||
Linux/glibc 部署可选择更积极地归还请求结束后的空闲内存，适合内存紧张且请求后 RSS 长时间偏高的机器。
|
||||
该选项默认关闭，官方 Linux 镜像支持；更频繁的映射与回收可能增加 CPU 开销，应结合实际负载观察
|
||||
|
||||
@@ -71,7 +86,7 @@ services:
|
||||
```
|
||||
|
||||
Compose 从 `services.app-runtime` 取得参数；不开启优化时保留该段并将值设为 `''`。
|
||||
PostgreSQL 和 Redis 的凭据桥接配置与该段并列，无需在 `compose.yaml` 重复填写参数
|
||||
PostgreSQL 和 Redis 的凭据桥接配置与该段并列，无需在 `compose.yaml` 重复填写 `GLIBC_TUNABLES`
|
||||
|
||||
校验并重建应用容器使配置生效；`docker compose restart` 不会更新容器环境变量：
|
||||
|
||||
@@ -94,6 +109,114 @@ GLIBC_TUNABLES=glibc.malloc.mmap_threshold=131072:glibc.malloc.trim_threshold=13
|
||||
该变量由 glibc 在进程启动时读取，不支持管理端热更新。
|
||||
它不设置内存上限，不释放仍被请求或连接持有的缓冲，也不处理容器文件缓存；观察时分别比较进程 RSS、cgroup 的 `anon` 与 `file`
|
||||
|
||||
#### PostgreSQL 与应用连接池
|
||||
|
||||
在 `deploy/config.yaml` 的 `store` 下设置连接池上限，未列出的池参数保留默认值：
|
||||
|
||||
```yaml
|
||||
store:
|
||||
pool:
|
||||
max_connections: 5
|
||||
```
|
||||
|
||||
应用默认上限为 20，允许的最小值为 2；降低上限可以限制实际数据库连接数，但也会增加请求、后台写入与管理查询之间的等待。
|
||||
如果出现连接获取超时，应结合数据库负载调大池容量，不要仅延长等待时间
|
||||
|
||||
将 `deploy/compose.yaml` 中 `services.postgres.command` 替换为以下列表，保留 UTC 参数：
|
||||
|
||||
```yaml
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- timezone=UTC
|
||||
- -c
|
||||
- log_timezone=UTC
|
||||
- -c
|
||||
- shared_buffers=64MB
|
||||
- -c
|
||||
- max_connections=30
|
||||
- -c
|
||||
- work_mem=2MB
|
||||
- -c
|
||||
- maintenance_work_mem=32MB
|
||||
- -c
|
||||
- autovacuum_work_mem=16MB
|
||||
- -c
|
||||
- max_parallel_workers_per_gather=0
|
||||
```
|
||||
|
||||
`shared_buffers` 缩小共享缓冲；服务端 `max_connections` 限制整个数据库的连接数，应覆盖所有应用池，并为备份、维护和其他客户端留余量。
|
||||
它与应用连接池上限是两个配置，调小服务端上限不会主动回收应用池里的空闲连接
|
||||
|
||||
`work_mem` 是每个排序或哈希操作的基础预算，哈希还受 `hash_mem_multiplier` 影响，一条 SQL 可以同时使用多份预算。
|
||||
`maintenance_work_mem` 限制维护操作内存，`autovacuum_work_mem` 按自动清理进程分别计；保持自动清理开启。
|
||||
关闭并行查询、缩小这些预算可能增加临时文件 I/O，并延长统计、索引创建与维护时间
|
||||
|
||||
参数定义见 [PostgreSQL 18 内存与并行查询](https://www.postgresql.org/docs/18/runtime-config-resource.html)和[连接设置](https://www.postgresql.org/docs/18/runtime-config-connection.html)
|
||||
|
||||
#### Redis 持久化与内存预算
|
||||
|
||||
项目默认开启 AOF，未显式设置 `maxmemory` 和定时 RDB 快照规则。
|
||||
小内存部署可保留 AOF，关闭定时 RDB 快照以减少快照期间的 fork / 写时复制峰值；代价是少一份独立的 RDB 快照，AOF 重写仍需要预留额外内存
|
||||
|
||||
在已有 `deploy/config.yaml` 中替换 `services.redis-credentials.command`，复用同一文件内的密码锚点，保留该服务的 `healthcheck`：
|
||||
|
||||
```yaml
|
||||
services:
|
||||
redis-credentials:
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- 'yes'
|
||||
- --save
|
||||
- ''
|
||||
- --requirepass
|
||||
- *redis_password
|
||||
# 当前用量和负载峰值明显低于 64 MiB 时，再取消以下注释
|
||||
# - --maxmemory
|
||||
# - 64mb
|
||||
# - --maxmemory-policy
|
||||
# - noeviction
|
||||
```
|
||||
|
||||
`--save` 后的空字符串必须保留，它表示关闭定时快照。持久化取舍见 [Redis 文档](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/)
|
||||
|
||||
设置内存上限前先查看 `INFO memory` 的 `used_memory`、`used_memory_peak` 和 `used_memory_rss`，为业务增长、分配器和持久化峰值留余量。
|
||||
`maxmemory 64mb` 只是可选起点，未达到上限时不会因此降低 RSS，也不是进程或容器的总内存上限。
|
||||
达到上限后，`noeviction` 会拒绝需要增加内存的写入，可能影响登录与请求调度，应调大预算或减少负载
|
||||
|
||||
Redis 保存租约、冷却和登录会话等[协调状态](../docs/architecture.md#8-状态所有权)，不要使用 `allkeys-lru`、`volatile-lru` 等提前淘汰键的策略。
|
||||
预算与策略定义见 [Redis 内存限制](https://redis.io/docs/latest/develop/reference/eviction/)
|
||||
|
||||
#### 使配置生效与检查效果
|
||||
|
||||
只调整应用配置时，按[应用空闲内存回收](#应用空闲内存回收)中的命令重建应用容器。
|
||||
调整 PostgreSQL / Redis 启动参数需要短暂停服，在维护窗口执行；只改其中一个服务时，数据库重建命令只列出该服务：
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/compose.yaml config --quiet
|
||||
docker compose -f deploy/compose.yaml stop codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build --no-deps --force-recreate --wait postgres redis
|
||||
docker compose -f deploy/compose.yaml up -d --no-build --no-deps --force-recreate --wait codex-proxy-rs
|
||||
```
|
||||
|
||||
数据库健康检查通过后再执行最后一条命令；`restart` 不会更新容器启动命令或环境变量。
|
||||
外部数据库需通过其管理方式应用参数，不能用本地 Compose 命令代替
|
||||
|
||||
通过交互提示输入对应数据库密码，核对实际生效值与内存占用，避免将密码写入命令行：
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/compose.yaml exec postgres psql -U codex_proxy -d codex_proxy -W \
|
||||
-c 'SHOW shared_buffers; SHOW max_connections; SHOW work_mem; SHOW maintenance_work_mem; SHOW autovacuum_work_mem; SHOW max_parallel_workers_per_gather;'
|
||||
docker compose -f deploy/compose.yaml exec redis redis-cli --askpass CONFIG GET appendonly save maxmemory maxmemory-policy
|
||||
docker compose -f deploy/compose.yaml exec redis redis-cli --askpass INFO memory
|
||||
docker compose -f deploy/compose.yaml stats --no-stream
|
||||
```
|
||||
|
||||
在相同负载下比较空闲、请求峰值与后台统计 / 维护期间的占用，并检查实际模型请求、管理查询是否正常，有无连接获取超时、Redis OOM 或容器重启。
|
||||
不要把 PostgreSQL 各进程的 RSS 直接相加，共享页会重复计数；容器文件缓存也应与匿名内存区分。
|
||||
容器硬内存上限不能代替这些预算，过紧会导致 OOM；需要回退时恢复调整前的参数，按同样流程重建对应容器
|
||||
|
||||
## 部署结构
|
||||
|
||||
```mermaid
|
||||
@@ -190,6 +313,42 @@ store:
|
||||
`redis://u1@redis-host:6379/0`，其中主机名必须能从应用容器访问；只改 `config.yaml` 不会生效。
|
||||
默认 Compose Redis 服务使用密码认证，不会自动创建 ACL 用户
|
||||
|
||||
## Windows 二进制部署
|
||||
|
||||
发行包提供 Windows x64（`windows_amd64`）和 ARM64（`windows_arm64`）版本，按运行机器的 CPU 架构选择。
|
||||
需先准备可访问的 PostgreSQL 和 Redis，创建数据库及用户；使用数据库备份功能时，还需将匹配服务端版本的 `pg_dump` 加入 `PATH`
|
||||
|
||||
从 [Release 附件](https://github.com/zyycn/codex-proxy-rs/releases) 下载对应的
|
||||
`codex-proxy-rs_<version>_windows_<arch>.tar.gz`，解压后保留 `codex-proxy-rs.exe`、`web/dist`、`deploy` 和 `plugins/official` 的目录关系。
|
||||
在解压后的 `codex-proxy-rs` 目录中打开 PowerShell，首次安装时创建配置：
|
||||
|
||||
```powershell
|
||||
Copy-Item .\deploy\config.example.yaml .\deploy\config.yaml
|
||||
```
|
||||
|
||||
编辑 `deploy/config.yaml`，填写 `store.database`、`store.redis` 的连接地址和独立密码，以及 `admin.default_password`。
|
||||
密码格式沿用模板要求；将已有的 `api.asset_directory` 改为 `../web/dist`，在 `host` 下合并以下更新与重启配置：
|
||||
|
||||
```yaml
|
||||
host:
|
||||
system_update:
|
||||
deployment_mode: binary
|
||||
self_restart_enabled: true
|
||||
api:
|
||||
asset_directory: '../web/dist'
|
||||
```
|
||||
|
||||
从安装目录启动：
|
||||
|
||||
```powershell
|
||||
.\codex-proxy-rs.exe serve
|
||||
```
|
||||
|
||||
启动成功后访问 `http://127.0.0.1:8080`，使用配置中的管理员账号登录；在运行终端按 `Ctrl+C` 停止服务。
|
||||
官方构建支持管理端[在线更新、重启与回滚](#管理端在线更新)，安装目录与 `.runtime` 必须可写。
|
||||
手动更换版本时先停止进程并备份当前发行文件，再整体替换 EXE、`web/dist` 和 `plugins/official`，
|
||||
保留 `deploy/config.yaml` 与 `.runtime`，按新版本模板合并配置后重新启动
|
||||
|
||||
## 启动
|
||||
|
||||
```bash
|
||||
@@ -233,8 +392,9 @@ PostgreSQL/Redis 启动密码。日常校验使用 `config --quiet`
|
||||
位置覆盖只影响 OpenAI 请求中受支持的位置与时区信息，不改变上下文日期、真实出口 IP、epoch 时间戳或系统时区。
|
||||
`openai.residency` 和 `openai.account_routing_override` 是独立于客户端身份的启动配置，
|
||||
分别指定 `x-openai-internal-codex-residency` 和 `x-openai-account-routing-override` 请求头。
|
||||
两项均为可选字符串，填写值原样发送，不校验地域或路由枚举；省略或设为 `null` 时删除对应的下游请求头且不补发。
|
||||
示例配置均填写 `us`，修改后重启应用生效。身份与位置字段见 [运行设置 API](../docs/api.md#8-运行设置)
|
||||
两项均为可选字符串，配置值原样发送并覆盖客户端同名头，支持自定义地域或路由值。
|
||||
省略或设为 `null` 时删除对应请求头。
|
||||
修改后重启应用生效。身份与位置字段见 [运行设置 API](../docs/api.md#8-运行设置)
|
||||
后台账号操作使用 Provider 自己的官方画像；发布资料刷新不改写用户选择或 `config.yaml`
|
||||
|
||||
### CPR 互联
|
||||
@@ -783,7 +943,7 @@ Release 必须提供当前 OS/架构的 `codex-proxy-rs_<version>_<os>_<arch>.ta
|
||||
宿主范围或能力版本不匹配时，可在插件页确认风险后尝试本次启动；清单、RPC、平台或包完整性错误不能绕过。具体操作见[插件故障处理](../docs/plugins.md#故障处理)
|
||||
|
||||
部署须保留完整的二进制、Web 资源与官方插件目录；不要手动修改制品元数据或迁移 checksum 绕过校验。
|
||||
macOS arm64 提供构建产物；部署前需验证目标平台的插件运行与上游调用，不能只以打包成功判断可用
|
||||
macOS arm64、Windows x64 / ARM64 提供构建产物；插件安装包必须匹配宿主 OS 和架构，部署前需验证目标平台的插件运行与上游调用，不能只以打包成功判断可用
|
||||
|
||||
## 备份与恢复
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ services:
|
||||
service: postgres-credentials
|
||||
image: postgres:18-bookworm@sha256:bbf83594798e144bff365a9c191c5d779f181fb3135902064ab8ff11918020fc
|
||||
restart: unless-stopped
|
||||
# 小内存部署可在此调整数据库预算，保留 UTC 参数，详见 README.md#小内存优化
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
|
||||
@@ -53,6 +53,7 @@ store:
|
||||
# 与 PostgreSQL 共用密码格式要求，并满足 Redis 服务端密码策略
|
||||
password: &redis_password ''
|
||||
# 可选：PostgreSQL 连接池预算；下列值与当前推荐默认一致。
|
||||
# 小内存低并发部署的池上限与数据库调优见 deploy/README.md#小内存优化
|
||||
# pool:
|
||||
# max_connections: 20
|
||||
# acquire_timeout_seconds: 5
|
||||
@@ -104,6 +105,7 @@ services:
|
||||
POSTGRES_PASSWORD: *postgres_password
|
||||
|
||||
redis-credentials:
|
||||
# 可选的持久化与内存预算见 deploy/README.md#小内存优化，调整时保留密码参数
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user