feat(openai): make the WebSocket fast-path budget configurable (#747)

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-11-19 17:10:41 +08:00
committed by GitHub
co-authored by Zyy Codex
parent 0922f3486c
commit d62868b692
10 changed files with 123 additions and 43 deletions
+15 -7
View File
@@ -8,7 +8,9 @@ use url::Url;
use crate::credential::CodexQuotaRefreshPolicy;
use crate::transport::session::{CodexSessionIdentity, CodexSessionIdentityError};
use crate::transport::websocket::CodexWebSocketPoolConfig;
use crate::transport::websocket::{
CodexWebSocketPoolConfig, DEFAULT_FAST_PATH_BUDGET_MS, WEBSOCKET_CONNECT_TIMEOUT,
};
use crate::{
OFFICIAL_CODEX_BASE_URL,
credential::token_client::{
@@ -165,26 +167,24 @@ impl CodexApiConfig {
/// Codex Responses WebSocket pool 的 Provider-owned 启动设置
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
#[serde(default)]
pub struct CodexWebSocketPoolSettings {
pub enabled: bool,
pub max_age_ms: u64,
pub max_connecting: usize,
#[serde(default = "default_websocket_pool_idle_timeout_ms")]
pub fast_path_budget_ms: u64,
pub idle_timeout_ms: u64,
pub stream_idle_timeout_ms: u64,
}
const fn default_websocket_pool_idle_timeout_ms() -> u64 {
300_000
}
impl Default for CodexWebSocketPoolSettings {
fn default() -> Self {
Self {
enabled: true,
max_age_ms: 55 * 60 * 1000,
max_connecting: 8,
idle_timeout_ms: default_websocket_pool_idle_timeout_ms(),
fast_path_budget_ms: DEFAULT_FAST_PATH_BUDGET_MS,
idle_timeout_ms: 300_000,
stream_idle_timeout_ms: 300_000,
}
}
@@ -192,6 +192,13 @@ impl Default for CodexWebSocketPoolSettings {
impl CodexWebSocketPoolSettings {
fn validate(&self) -> Result<(), OpenAiConfigError> {
if self.fast_path_budget_ms == 0
|| Duration::from_millis(self.fast_path_budget_ms) > WEBSOCKET_CONNECT_TIMEOUT
{
return Err(OpenAiConfigError::InvalidField(
"openai.ws_pool.fast_path_budget_ms",
));
}
if self.max_age_ms == 0 || self.max_connecting == 0 || self.idle_timeout_ms == 0 {
return Err(OpenAiConfigError::InvalidField("openai.ws_pool"));
}
@@ -203,6 +210,7 @@ impl CodexWebSocketPoolSettings {
enabled: self.enabled,
max_age: Duration::from_millis(self.max_age_ms),
max_connecting: self.max_connecting,
fast_path_budget: Duration::from_millis(self.fast_path_budget_ms),
idle_timeout: Duration::from_millis(self.idle_timeout_ms),
stream_idle_timeout: (self.stream_idle_timeout_ms != 0)
.then(|| Duration::from_millis(self.stream_idle_timeout_ms)),
@@ -2,7 +2,10 @@
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext, diagnostic_json};
use std::{sync::Arc, time::Instant};
use std::{
sync::Arc,
time::{Duration, Instant},
};
use futures::{StreamExt, TryStreamExt};
use gateway_protocol::openai::{
@@ -32,9 +35,10 @@ use crate::transport::{
},
response_meta,
websocket::{
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPoolKey,
DEFAULT_STREAM_IDLE_TIMEOUT, PreviousResponseUnavailableReason, WEBSOCKET_FAST_PATH_BUDGET,
WebSocketFastPath, execute_prepared_response_create_request_stream, post_send_ambiguous,
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
CodexWebSocketPoolKey, DEFAULT_FAST_PATH_BUDGET_MS, DEFAULT_STREAM_IDLE_TIMEOUT,
PreviousResponseUnavailableReason, WebSocketFastPath,
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool, websocket_audit_dir,
write_websocket_audit_artifact_from_env,
},
@@ -375,7 +379,10 @@ impl CodexBackendClient {
.map(|pool| (pool, pool_key, downstream_cancellation));
let fast_path_budget = match requirement {
TransportRequirement::PersistedContinuation | TransportRequirement::NewChain => {
Some(WEBSOCKET_FAST_PATH_BUDGET)
Some(self.websocket_pool.as_deref().map_or(
Duration::from_millis(DEFAULT_FAST_PATH_BUDGET_MS),
CodexWebSocketPool::fast_path_budget,
))
}
TransportRequirement::ExplicitWebSocketWarmup
| TransportRequirement::WebSocketNewChain
@@ -32,8 +32,6 @@ use super::{
pump::{PumpLogContext, PumpedWebSocket, transport_metric_reason},
};
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
/// WebSocket 快路径的控制流结果;预算未命中不代表连接失败
pub(crate) enum WebSocketFastPath<T> {
/// 在前台预算内获得结果
@@ -575,7 +573,7 @@ async fn wait_for_shared_connect(
wait_for_fast_path(waiter.started_at(), fast_path_budget, waiter.wait()).await
}
// 同一次 opening 的所有等待者共享截止时间;后来的请求不会重置 800ms 预算
// 同一次 opening 的所有等待者共享截止时间
async fn wait_for_fast_path<F: Future>(
started_at: TokioInstant,
budget: Option<Duration>,
@@ -35,7 +35,7 @@ use super::{
use super::control::WEBSOCKET_SEND_TIMEOUT;
const WEBSOCKET_EXTENSIONS: &str = "permessage-deflate; client_max_window_bits";
const WEBSOCKET_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
pub(crate) const WEBSOCKET_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
impl CodexWebSocketConnection {
/// 复用传输库的握手编码检查,不能在拨号后才发现插件输出不可表示
@@ -39,11 +39,11 @@ pub use self::{
};
pub(crate) use self::{
coordinator::{
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool,
PreparedWebSocket, WebSocketFastPath, execute_prepared_response_create_request_stream,
post_send_ambiguous, prepare_response_create_request_with_pool,
},
error::WEBSOCKET_CLOSE_MESSAGE_TOO_BIG,
handshake::WEBSOCKET_CONNECT_TIMEOUT,
live::{connect_live_sideband, into_live_relay},
pool::DEFAULT_STREAM_IDLE_TIMEOUT,
pool::{DEFAULT_FAST_PATH_BUDGET_MS, DEFAULT_STREAM_IDLE_TIMEOUT},
};
@@ -37,6 +37,7 @@ pub(crate) use self::{
};
const DEFAULT_MAX_CONNECTING: usize = 16;
pub(crate) const DEFAULT_FAST_PATH_BUDGET_MS: u64 = 800;
const DEFAULT_MAX_AGE: Duration = Duration::from_mins(55);
const DEFAULT_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(5);
const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
@@ -70,6 +71,8 @@ pub struct CodexWebSocketPoolConfig {
pub max_age: Duration,
/// 所有账号合计允许并发执行的 opening 数
pub max_connecting: usize,
/// 允许回退 HTTP 的请求在前台等待 WebSocket 建连的时长
pub fast_path_budget: Duration,
/// 后台维护间隔;`None` 表示不启动后台任务
pub maintenance_interval: Option<Duration>,
/// 未绑定下游 WS 的连接归池后可空闲的时间,不由 Ping/Pong 刷新
@@ -84,6 +87,7 @@ impl Default for CodexWebSocketPoolConfig {
enabled: true,
max_age: DEFAULT_MAX_AGE,
max_connecting: DEFAULT_MAX_CONNECTING,
fast_path_budget: Duration::from_millis(DEFAULT_FAST_PATH_BUDGET_MS),
maintenance_interval: Some(DEFAULT_MAINTENANCE_INTERVAL),
idle_timeout: DEFAULT_IDLE_TIMEOUT,
stream_idle_timeout: Some(DEFAULT_STREAM_IDLE_TIMEOUT),
@@ -117,6 +121,10 @@ impl CodexWebSocketPool {
pool
}
pub(crate) fn fast_path_budget(&self) -> Duration {
self.config.fast_path_budget
}
/// 等待下一条上游消息的空闲超时;`None` 或零值使用默认 300 秒
pub(crate) fn stream_idle_timeout(&self) -> Option<Duration> {
self.config.stream_idle_timeout
+40 -13
View File
@@ -96,26 +96,53 @@ fn openai_stream_retry_budget_uses_the_official_hard_cap() {
}
#[test]
fn websocket_pool_idle_timeout_accepts_existing_configs_and_explicit_overrides() {
for (idle_timeout_ms, expected) in [(None, 300_000), (Some(600_000), 600_000)] {
let mut document = serde_json::json!({
"ws_pool": {
fn websocket_pool_timeouts_accept_defaults_and_individual_overrides() {
for (settings, idle_ms, fast_ms) in [
(
serde_json::json!({
"enabled": true,
"max_age_ms": 3_300_000,
"max_connecting": 8,
"stream_idle_timeout_ms": 300_000
}
});
if let Some(idle_timeout_ms) = idle_timeout_ms {
document["ws_pool"]["idle_timeout_ms"] = serde_json::json!(idle_timeout_ms);
}
let mut config: OpenAiConfig = serde_json::from_value(document).unwrap();
}),
300_000,
800,
),
(
serde_json::json!({"idle_timeout_ms": 600_000}),
600_000,
800,
),
(serde_json::json!({"fast_path_budget_ms": 1}), 300_000, 1),
(
serde_json::json!({"fast_path_budget_ms": 15_000}),
300_000,
15_000,
),
] {
let mut config: OpenAiConfig =
serde_json::from_value(serde_json::json!({"ws_pool": settings})).unwrap();
config
.resolve_and_validate(Path::new("/srv/gateway"))
.unwrap();
assert_eq!(
config.websocket_pool_config().idle_timeout,
Duration::from_millis(expected)
let pool = config.websocket_pool_config();
assert_eq!(pool.idle_timeout, Duration::from_millis(idle_ms));
assert_eq!(pool.fast_path_budget, Duration::from_millis(fast_ms));
}
}
#[test]
fn websocket_fast_path_budget_must_fit_the_opening_timeout() {
for budget_ms in [0, 15_001] {
let mut config = valid_config();
config.ws_pool.fast_path_budget_ms = budget_ms;
let error = config
.resolve_and_validate(Path::new("/srv/gateway"))
.unwrap_err();
assert!(
error
.to_string()
.contains("openai.ws_pool.fast_path_budget_ms")
);
}
}
@@ -341,6 +341,31 @@ async fn downstream_websocket_new_chain_should_preserve_continuation_after_slow_
#[tokio::test]
async fn external_continuation_should_wait_for_a_cold_websocket() {
assert_slow_opening_uses_websocket(
external_continuation_request("conversation-cross-account"),
CodexWebSocketPoolConfig::default(),
)
.await;
}
#[tokio::test]
async fn configured_fast_path_budget_should_allow_slow_websocket_openings() {
let config = CodexWebSocketPoolConfig {
fast_path_budget: Duration::from_secs(3),
..Default::default()
};
let mut persisted = new_chain_request("persisted");
persisted.set_previous_response_id(Some("resp_persisted".into()));
persisted.previous_response_scope = Some(PreviousResponseScope::Persisted);
for request in [new_chain_request("new-chain"), persisted] {
assert_slow_opening_uses_websocket(request, config).await;
}
}
async fn assert_slow_opening_uses_websocket(
request: CodexResponsesRequest,
config: CodexWebSocketPoolConfig,
) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
@@ -350,12 +375,12 @@ async fn external_continuation_should_wait_for_a_cold_websocket() {
let _payload = websocket.next().await.unwrap().unwrap();
websocket
.send(Message::Text(
completed_websocket_response("resp_cross_account", 2, 1).into(),
completed_websocket_response("resp_slow_opening", 2, 1).into(),
))
.await
.unwrap();
});
let pool = Arc::new(CodexWebSocketPool::default());
let pool = Arc::new(CodexWebSocketPool::with_config(config));
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
@@ -363,13 +388,16 @@ async fn external_continuation_should_wait_for_a_cold_websocket() {
)
.with_websocket_pool(Arc::clone(&pool));
let response = backend
.create_response(
&external_continuation_request("conversation-cross-account"),
request_context("req_cross_account", Some("chatgpt-account-b")),
)
.await
.expect("external continuation should complete over WebSocket");
let response = timeout(
Duration::from_secs(5),
backend.create_response(
&request,
request_context("req_slow_opening", Some("chatgpt-account-b")),
),
)
.await
.expect("slow WebSocket opening should complete")
.unwrap();
server.await.unwrap();
assert_eq!(response.transport, CodexBackendTransport::WebSocket);
+3
View File
@@ -544,6 +544,9 @@ OpenAI Responses 上游 WebSocket 每 25 秒发送 Ping 保活,并回应上游
修改上限对后续请求生效,已存在的连接继续计数;调低上限不会强制中断在途或绑定下游 WS 的连接。
预算仅覆盖宿主原生 OpenAI Responses 连接,插件自建连接及其他 HTTP/WS 流量仍需预留容量
允许回退 HTTP 的 Responses 新链和持久化续接,由 `openai.ws_pool.fast_path_budget_ms` 控制前台等待上游 WS 建连的时长,默认 800 毫秒,可设为 1–15000 毫秒,重启应用后生效。
超出预算时,本次请求走 HTTP/SSE,已启动的池化建连在后台继续。高延迟出口可调大该值,代价是仍需回退的请求会等得更久
未绑定活跃客户端 WS 的池化连接,归池后按 `openai.ws_pool.idle_timeout_ms` 计时,默认 `300000`(5 分钟,必须大于零)。
Ping/Pong 不延长空闲期限;默认每 5 秒扫描,通常在空闲 300~305 秒开始回收,复用前也检查是否到期。
关闭先停止复用,最多等待 1 秒发送 Close,随后终止并等待 socket 释放,容量在物理连接释放后才归还。
+2 -1
View File
@@ -85,11 +85,12 @@ openai:
residency: us
# 账号路由参考值:us、us_cr
account_routing_override: null
# 可选:完整的 Responses 上游 WS 池配置;省略整段使用以下默认值。
# Responses 上游 WS 池配置,可单独覆盖下列默认值
# ws_pool:
# enabled: true
# max_age_ms: 3300000
# max_connecting: 8
# fast_path_budget_ms: 800 # 回退 HTTP 前等待 WS 建连的时长,范围 1–15000 毫秒
# idle_timeout_ms: 300000 # 仅回收未绑定活跃客户端 WS 的空闲连接,必须大于零。
# stream_idle_timeout_ms: 300000