mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-22 00:00:37 +00:00
feat(openai): make the WebSocket fast-path budget configurable (#747)
Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -8,7 +8,9 @@ use url::Url;
|
||||
|
||||
use crate::credential::CodexQuotaRefreshPolicy;
|
||||
use crate::transport::session::{CodexSessionIdentity, CodexSessionIdentityError};
|
||||
use crate::transport::websocket::CodexWebSocketPoolConfig;
|
||||
use crate::transport::websocket::{
|
||||
CodexWebSocketPoolConfig, DEFAULT_FAST_PATH_BUDGET_MS, WEBSOCKET_CONNECT_TIMEOUT,
|
||||
};
|
||||
use crate::{
|
||||
OFFICIAL_CODEX_BASE_URL,
|
||||
credential::token_client::{
|
||||
@@ -165,26 +167,24 @@ impl CodexApiConfig {
|
||||
|
||||
/// Codex Responses WebSocket pool 的 Provider-owned 启动设置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct CodexWebSocketPoolSettings {
|
||||
pub enabled: bool,
|
||||
pub max_age_ms: u64,
|
||||
pub max_connecting: usize,
|
||||
#[serde(default = "default_websocket_pool_idle_timeout_ms")]
|
||||
pub fast_path_budget_ms: u64,
|
||||
pub idle_timeout_ms: u64,
|
||||
pub stream_idle_timeout_ms: u64,
|
||||
}
|
||||
|
||||
const fn default_websocket_pool_idle_timeout_ms() -> u64 {
|
||||
300_000
|
||||
}
|
||||
|
||||
impl Default for CodexWebSocketPoolSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
max_age_ms: 55 * 60 * 1000,
|
||||
max_connecting: 8,
|
||||
idle_timeout_ms: default_websocket_pool_idle_timeout_ms(),
|
||||
fast_path_budget_ms: DEFAULT_FAST_PATH_BUDGET_MS,
|
||||
idle_timeout_ms: 300_000,
|
||||
stream_idle_timeout_ms: 300_000,
|
||||
}
|
||||
}
|
||||
@@ -192,6 +192,13 @@ impl Default for CodexWebSocketPoolSettings {
|
||||
|
||||
impl CodexWebSocketPoolSettings {
|
||||
fn validate(&self) -> Result<(), OpenAiConfigError> {
|
||||
if self.fast_path_budget_ms == 0
|
||||
|| Duration::from_millis(self.fast_path_budget_ms) > WEBSOCKET_CONNECT_TIMEOUT
|
||||
{
|
||||
return Err(OpenAiConfigError::InvalidField(
|
||||
"openai.ws_pool.fast_path_budget_ms",
|
||||
));
|
||||
}
|
||||
if self.max_age_ms == 0 || self.max_connecting == 0 || self.idle_timeout_ms == 0 {
|
||||
return Err(OpenAiConfigError::InvalidField("openai.ws_pool"));
|
||||
}
|
||||
@@ -203,6 +210,7 @@ impl CodexWebSocketPoolSettings {
|
||||
enabled: self.enabled,
|
||||
max_age: Duration::from_millis(self.max_age_ms),
|
||||
max_connecting: self.max_connecting,
|
||||
fast_path_budget: Duration::from_millis(self.fast_path_budget_ms),
|
||||
idle_timeout: Duration::from_millis(self.idle_timeout_ms),
|
||||
stream_idle_timeout: (self.stream_idle_timeout_ms != 0)
|
||||
.then(|| Duration::from_millis(self.stream_idle_timeout_ms)),
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext, diagnostic_json};
|
||||
|
||||
use std::{sync::Arc, time::Instant};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use gateway_protocol::openai::{
|
||||
@@ -32,9 +35,10 @@ use crate::transport::{
|
||||
},
|
||||
response_meta,
|
||||
websocket::{
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPoolKey,
|
||||
DEFAULT_STREAM_IDLE_TIMEOUT, PreviousResponseUnavailableReason, WEBSOCKET_FAST_PATH_BUDGET,
|
||||
WebSocketFastPath, execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
|
||||
CodexWebSocketPoolKey, DEFAULT_FAST_PATH_BUDGET_MS, DEFAULT_STREAM_IDLE_TIMEOUT,
|
||||
PreviousResponseUnavailableReason, WebSocketFastPath,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool, websocket_audit_dir,
|
||||
write_websocket_audit_artifact_from_env,
|
||||
},
|
||||
@@ -375,7 +379,10 @@ impl CodexBackendClient {
|
||||
.map(|pool| (pool, pool_key, downstream_cancellation));
|
||||
let fast_path_budget = match requirement {
|
||||
TransportRequirement::PersistedContinuation | TransportRequirement::NewChain => {
|
||||
Some(WEBSOCKET_FAST_PATH_BUDGET)
|
||||
Some(self.websocket_pool.as_deref().map_or(
|
||||
Duration::from_millis(DEFAULT_FAST_PATH_BUDGET_MS),
|
||||
CodexWebSocketPool::fast_path_budget,
|
||||
))
|
||||
}
|
||||
TransportRequirement::ExplicitWebSocketWarmup
|
||||
| TransportRequirement::WebSocketNewChain
|
||||
|
||||
@@ -32,8 +32,6 @@ use super::{
|
||||
pump::{PumpLogContext, PumpedWebSocket, transport_metric_reason},
|
||||
};
|
||||
|
||||
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
|
||||
|
||||
/// WebSocket 快路径的控制流结果;预算未命中不代表连接失败
|
||||
pub(crate) enum WebSocketFastPath<T> {
|
||||
/// 在前台预算内获得结果
|
||||
@@ -575,7 +573,7 @@ async fn wait_for_shared_connect(
|
||||
wait_for_fast_path(waiter.started_at(), fast_path_budget, waiter.wait()).await
|
||||
}
|
||||
|
||||
// 同一次 opening 的所有等待者共享截止时间;后来的请求不会重置 800ms 预算
|
||||
// 同一次 opening 的所有等待者共享截止时间
|
||||
async fn wait_for_fast_path<F: Future>(
|
||||
started_at: TokioInstant,
|
||||
budget: Option<Duration>,
|
||||
|
||||
@@ -35,7 +35,7 @@ use super::{
|
||||
use super::control::WEBSOCKET_SEND_TIMEOUT;
|
||||
|
||||
const WEBSOCKET_EXTENSIONS: &str = "permessage-deflate; client_max_window_bits";
|
||||
const WEBSOCKET_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
pub(crate) const WEBSOCKET_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
impl CodexWebSocketConnection {
|
||||
/// 复用传输库的握手编码检查,不能在拨号后才发现插件输出不可表示
|
||||
|
||||
@@ -39,11 +39,11 @@ pub use self::{
|
||||
};
|
||||
pub(crate) use self::{
|
||||
coordinator::{
|
||||
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool,
|
||||
PreparedWebSocket, WebSocketFastPath, execute_prepared_response_create_request_stream,
|
||||
post_send_ambiguous, prepare_response_create_request_with_pool,
|
||||
},
|
||||
error::WEBSOCKET_CLOSE_MESSAGE_TOO_BIG,
|
||||
handshake::WEBSOCKET_CONNECT_TIMEOUT,
|
||||
live::{connect_live_sideband, into_live_relay},
|
||||
pool::DEFAULT_STREAM_IDLE_TIMEOUT,
|
||||
pool::{DEFAULT_FAST_PATH_BUDGET_MS, DEFAULT_STREAM_IDLE_TIMEOUT},
|
||||
};
|
||||
|
||||
@@ -37,6 +37,7 @@ pub(crate) use self::{
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_CONNECTING: usize = 16;
|
||||
pub(crate) const DEFAULT_FAST_PATH_BUDGET_MS: u64 = 800;
|
||||
const DEFAULT_MAX_AGE: Duration = Duration::from_mins(55);
|
||||
const DEFAULT_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
@@ -70,6 +71,8 @@ pub struct CodexWebSocketPoolConfig {
|
||||
pub max_age: Duration,
|
||||
/// 所有账号合计允许并发执行的 opening 数
|
||||
pub max_connecting: usize,
|
||||
/// 允许回退 HTTP 的请求在前台等待 WebSocket 建连的时长
|
||||
pub fast_path_budget: Duration,
|
||||
/// 后台维护间隔;`None` 表示不启动后台任务
|
||||
pub maintenance_interval: Option<Duration>,
|
||||
/// 未绑定下游 WS 的连接归池后可空闲的时间,不由 Ping/Pong 刷新
|
||||
@@ -84,6 +87,7 @@ impl Default for CodexWebSocketPoolConfig {
|
||||
enabled: true,
|
||||
max_age: DEFAULT_MAX_AGE,
|
||||
max_connecting: DEFAULT_MAX_CONNECTING,
|
||||
fast_path_budget: Duration::from_millis(DEFAULT_FAST_PATH_BUDGET_MS),
|
||||
maintenance_interval: Some(DEFAULT_MAINTENANCE_INTERVAL),
|
||||
idle_timeout: DEFAULT_IDLE_TIMEOUT,
|
||||
stream_idle_timeout: Some(DEFAULT_STREAM_IDLE_TIMEOUT),
|
||||
@@ -117,6 +121,10 @@ impl CodexWebSocketPool {
|
||||
pool
|
||||
}
|
||||
|
||||
pub(crate) fn fast_path_budget(&self) -> Duration {
|
||||
self.config.fast_path_budget
|
||||
}
|
||||
|
||||
/// 等待下一条上游消息的空闲超时;`None` 或零值使用默认 300 秒
|
||||
pub(crate) fn stream_idle_timeout(&self) -> Option<Duration> {
|
||||
self.config.stream_idle_timeout
|
||||
|
||||
@@ -96,26 +96,53 @@ fn openai_stream_retry_budget_uses_the_official_hard_cap() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_pool_idle_timeout_accepts_existing_configs_and_explicit_overrides() {
|
||||
for (idle_timeout_ms, expected) in [(None, 300_000), (Some(600_000), 600_000)] {
|
||||
let mut document = serde_json::json!({
|
||||
"ws_pool": {
|
||||
fn websocket_pool_timeouts_accept_defaults_and_individual_overrides() {
|
||||
for (settings, idle_ms, fast_ms) in [
|
||||
(
|
||||
serde_json::json!({
|
||||
"enabled": true,
|
||||
"max_age_ms": 3_300_000,
|
||||
"max_connecting": 8,
|
||||
"stream_idle_timeout_ms": 300_000
|
||||
}
|
||||
});
|
||||
if let Some(idle_timeout_ms) = idle_timeout_ms {
|
||||
document["ws_pool"]["idle_timeout_ms"] = serde_json::json!(idle_timeout_ms);
|
||||
}
|
||||
let mut config: OpenAiConfig = serde_json::from_value(document).unwrap();
|
||||
}),
|
||||
300_000,
|
||||
800,
|
||||
),
|
||||
(
|
||||
serde_json::json!({"idle_timeout_ms": 600_000}),
|
||||
600_000,
|
||||
800,
|
||||
),
|
||||
(serde_json::json!({"fast_path_budget_ms": 1}), 300_000, 1),
|
||||
(
|
||||
serde_json::json!({"fast_path_budget_ms": 15_000}),
|
||||
300_000,
|
||||
15_000,
|
||||
),
|
||||
] {
|
||||
let mut config: OpenAiConfig =
|
||||
serde_json::from_value(serde_json::json!({"ws_pool": settings})).unwrap();
|
||||
config
|
||||
.resolve_and_validate(Path::new("/srv/gateway"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config.websocket_pool_config().idle_timeout,
|
||||
Duration::from_millis(expected)
|
||||
let pool = config.websocket_pool_config();
|
||||
assert_eq!(pool.idle_timeout, Duration::from_millis(idle_ms));
|
||||
assert_eq!(pool.fast_path_budget, Duration::from_millis(fast_ms));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_fast_path_budget_must_fit_the_opening_timeout() {
|
||||
for budget_ms in [0, 15_001] {
|
||||
let mut config = valid_config();
|
||||
config.ws_pool.fast_path_budget_ms = budget_ms;
|
||||
let error = config
|
||||
.resolve_and_validate(Path::new("/srv/gateway"))
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("openai.ws_pool.fast_path_budget_ms")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -341,6 +341,31 @@ async fn downstream_websocket_new_chain_should_preserve_continuation_after_slow_
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
assert_slow_opening_uses_websocket(
|
||||
external_continuation_request("conversation-cross-account"),
|
||||
CodexWebSocketPoolConfig::default(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configured_fast_path_budget_should_allow_slow_websocket_openings() {
|
||||
let config = CodexWebSocketPoolConfig {
|
||||
fast_path_budget: Duration::from_secs(3),
|
||||
..Default::default()
|
||||
};
|
||||
let mut persisted = new_chain_request("persisted");
|
||||
persisted.set_previous_response_id(Some("resp_persisted".into()));
|
||||
persisted.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
for request in [new_chain_request("new-chain"), persisted] {
|
||||
assert_slow_opening_uses_websocket(request, config).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_slow_opening_uses_websocket(
|
||||
request: CodexResponsesRequest,
|
||||
config: CodexWebSocketPoolConfig,
|
||||
) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
@@ -350,12 +375,12 @@ async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
let _payload = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_cross_account", 2, 1).into(),
|
||||
completed_websocket_response("resp_slow_opening", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let pool = Arc::new(CodexWebSocketPool::with_config(config));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
@@ -363,13 +388,16 @@ async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
|
||||
let response = backend
|
||||
.create_response(
|
||||
&external_continuation_request("conversation-cross-account"),
|
||||
request_context("req_cross_account", Some("chatgpt-account-b")),
|
||||
)
|
||||
.await
|
||||
.expect("external continuation should complete over WebSocket");
|
||||
let response = timeout(
|
||||
Duration::from_secs(5),
|
||||
backend.create_response(
|
||||
&request,
|
||||
request_context("req_slow_opening", Some("chatgpt-account-b")),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("slow WebSocket opening should complete")
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
|
||||
assert_eq!(response.transport, CodexBackendTransport::WebSocket);
|
||||
|
||||
@@ -544,6 +544,9 @@ OpenAI Responses 上游 WebSocket 每 25 秒发送 Ping 保活,并回应上游
|
||||
修改上限对后续请求生效,已存在的连接继续计数;调低上限不会强制中断在途或绑定下游 WS 的连接。
|
||||
预算仅覆盖宿主原生 OpenAI Responses 连接,插件自建连接及其他 HTTP/WS 流量仍需预留容量
|
||||
|
||||
允许回退 HTTP 的 Responses 新链和持久化续接,由 `openai.ws_pool.fast_path_budget_ms` 控制前台等待上游 WS 建连的时长,默认 800 毫秒,可设为 1–15000 毫秒,重启应用后生效。
|
||||
超出预算时,本次请求走 HTTP/SSE,已启动的池化建连在后台继续。高延迟出口可调大该值,代价是仍需回退的请求会等得更久
|
||||
|
||||
未绑定活跃客户端 WS 的池化连接,归池后按 `openai.ws_pool.idle_timeout_ms` 计时,默认 `300000`(5 分钟,必须大于零)。
|
||||
Ping/Pong 不延长空闲期限;默认每 5 秒扫描,通常在空闲 300~305 秒开始回收,复用前也检查是否到期。
|
||||
关闭先停止复用,最多等待 1 秒发送 Close,随后终止并等待 socket 释放,容量在物理连接释放后才归还。
|
||||
|
||||
@@ -85,11 +85,12 @@ openai:
|
||||
residency: us
|
||||
# 账号路由参考值:us、us_cr
|
||||
account_routing_override: null
|
||||
# 可选:完整的 Responses 上游 WS 池配置;省略整段使用以下默认值。
|
||||
# Responses 上游 WS 池配置,可单独覆盖下列默认值
|
||||
# ws_pool:
|
||||
# enabled: true
|
||||
# max_age_ms: 3300000
|
||||
# max_connecting: 8
|
||||
# fast_path_budget_ms: 800 # 回退 HTTP 前等待 WS 建连的时长,范围 1–15000 毫秒
|
||||
# idle_timeout_ms: 300000 # 仅回收未绑定活跃客户端 WS 的空闲连接,必须大于零。
|
||||
# stream_idle_timeout_ms: 300000
|
||||
|
||||
|
||||
Reference in New Issue
Block a user