mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-17 16:06:57 +00:00
fix(routing): skip unreachable account proxies for the rest of a request
This commit is contained in:
@@ -19,7 +19,7 @@ pub use quota_settings::{
|
||||
QuotaPauseDecision,
|
||||
};
|
||||
mod proxy;
|
||||
pub use proxy::{InvalidOutboundProxy, OutboundProxy};
|
||||
pub use proxy::{InvalidOutboundProxy, OutboundProxy, ProxyEndpoint};
|
||||
mod scheduling;
|
||||
pub mod scope;
|
||||
mod selection;
|
||||
|
||||
@@ -9,6 +9,30 @@ pub struct OutboundProxy {
|
||||
max_websocket_connections: u32,
|
||||
}
|
||||
|
||||
/// 拨号目标只按主机与端口识别;认证、协议与连接上限不改变 TCP 入口
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProxyEndpoint {
|
||||
host: String,
|
||||
port: u16,
|
||||
}
|
||||
|
||||
impl ProxyEndpoint {
|
||||
#[must_use]
|
||||
pub fn matches(&self, proxy: &OutboundProxy) -> bool {
|
||||
proxy.url.port_or_known_default() == Some(self.port)
|
||||
&& proxy
|
||||
.url
|
||||
.host_str()
|
||||
.is_some_and(|host| host.eq_ignore_ascii_case(&self.host))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProxyEndpoint {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ProxyEndpoint(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("invalid outbound proxy; expected http, https, socks5 or socks5h URL with a host and port")]
|
||||
pub struct InvalidOutboundProxy;
|
||||
@@ -51,6 +75,21 @@ impl OutboundProxy {
|
||||
self.url.as_str()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn dial_endpoint(&self) -> ProxyEndpoint {
|
||||
ProxyEndpoint {
|
||||
host: self
|
||||
.url
|
||||
.host_str()
|
||||
.expect("validated proxy host")
|
||||
.to_ascii_lowercase(),
|
||||
port: self
|
||||
.url
|
||||
.port_or_known_default()
|
||||
.expect("validated proxy port"),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn endpoint(&self) -> String {
|
||||
let mut url = self.url.clone();
|
||||
|
||||
@@ -367,6 +367,7 @@ where
|
||||
account_rotations: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
pending_retry: None,
|
||||
@@ -525,6 +526,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
account_rotations: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
excluded_proxy_endpoints: Vec<crate::account::ProxyEndpoint>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算
|
||||
@@ -1098,7 +1100,7 @@ where
|
||||
),
|
||||
}
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
let mut account_context = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(target) => AccountAttemptContext::diagnostic(
|
||||
self.excluded_accounts.clone(),
|
||||
target.clone(),
|
||||
@@ -1115,6 +1117,9 @@ where
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
}));
|
||||
account_context
|
||||
.excluded_proxy_endpoints
|
||||
.clone_from(&self.excluded_proxy_endpoints);
|
||||
let provider = self
|
||||
.engine
|
||||
.providers()
|
||||
@@ -1237,6 +1242,11 @@ where
|
||||
Err(error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& self
|
||||
.last_retryable_failure
|
||||
.as_ref()
|
||||
.and_then(ProviderError::unreachable_proxy)
|
||||
.is_none()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
@@ -2000,6 +2010,11 @@ where
|
||||
}
|
||||
StreamRetry::Pending(retry) => self.pending_retry = Some(retry),
|
||||
StreamRetry::ConnectionRotation(retry) => {
|
||||
if let Some(endpoint) = pending.error.unreachable_proxy()
|
||||
&& !self.excluded_proxy_endpoints.contains(endpoint)
|
||||
{
|
||||
self.excluded_proxy_endpoints.push(endpoint.clone());
|
||||
}
|
||||
self.excluded_accounts.insert(retry.account.clone());
|
||||
self.connection_rotation = Some(retry);
|
||||
}
|
||||
|
||||
@@ -124,6 +124,7 @@ impl AttemptTrigger {
|
||||
/// 一旦设置,Provider 在本 attempt 内不得换号或切换 target
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccountAttemptContext {
|
||||
excluded_proxy_endpoints: Vec<crate::account::ProxyEndpoint>,
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: Option<ProviderAccountId>,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
@@ -140,6 +141,7 @@ impl AccountAttemptContext {
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account,
|
||||
state_owner,
|
||||
@@ -160,6 +162,7 @@ impl AccountAttemptContext {
|
||||
) -> Self {
|
||||
let (required_account, eligibility) = target.into_selection();
|
||||
Self {
|
||||
excluded_proxy_endpoints: Vec::new(),
|
||||
excluded_accounts,
|
||||
required_account: Some(required_account),
|
||||
state_owner,
|
||||
@@ -896,6 +899,17 @@ impl AttemptContext {
|
||||
self.account.excluded_accounts()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn excludes_proxy(&self, proxy: Option<&crate::account::OutboundProxy>) -> bool {
|
||||
!self.account.excluded_proxy_endpoints.is_empty()
|
||||
&& proxy.is_some_and(|proxy| {
|
||||
self.account
|
||||
.excluded_proxy_endpoints
|
||||
.iter()
|
||||
.any(|endpoint| endpoint.matches(proxy))
|
||||
})
|
||||
}
|
||||
|
||||
/// 管理端诊断或请求局部恢复 attempt 强制使用的唯一账号
|
||||
#[must_use]
|
||||
pub const fn required_account(&self) -> Option<&ProviderAccountId> {
|
||||
|
||||
@@ -540,6 +540,7 @@ struct ProviderErrorUpstreamValues {
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorFailureObservation {
|
||||
unreachable_proxy: Option<crate::account::ProxyEndpoint>,
|
||||
continuation_unavailable_reason: Option<&'static str>,
|
||||
connection: Option<ProviderConnectionObservation>,
|
||||
}
|
||||
@@ -662,6 +663,20 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 仅当连接账号绑定代理的入口失败时记录;不用于持久化或输出地址
|
||||
#[must_use]
|
||||
pub fn with_unreachable_proxy(mut self, endpoint: crate::account::ProxyEndpoint) -> Self {
|
||||
self.failure_observation_mut().unreachable_proxy = Some(endpoint);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn unreachable_proxy(&self) -> Option<&crate::account::ProxyEndpoint> {
|
||||
self.failure_observation
|
||||
.as_deref()
|
||||
.and_then(|observation| observation.unreachable_proxy.as_ref())
|
||||
}
|
||||
|
||||
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
|
||||
self.failure_observation
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
|
||||
|
||||
@@ -1112,6 +1112,12 @@ impl CodexCredentialSelector {
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if !diagnostic {
|
||||
for candidate in &candidates {
|
||||
if request
|
||||
.attempt
|
||||
.excludes_proxy(candidate.account.outbound_proxy())
|
||||
{
|
||||
excluded.insert(candidate.account.id().clone());
|
||||
}
|
||||
if candidate.account.quota_paused() {
|
||||
if follow_only
|
||||
&& binding.as_ref().is_some_and(|binding| {
|
||||
@@ -1280,6 +1286,16 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
let Some(selection) = selection else {
|
||||
if follow_only && let Some(binding) = binding.as_ref() {
|
||||
// 如果整个候选池共用本请求已排除的入口,等待根线程迁移也没有可用出口
|
||||
if !candidates.is_empty()
|
||||
&& candidates.iter().all(|candidate| {
|
||||
request
|
||||
.attempt
|
||||
.excludes_proxy(candidate.account.outbound_proxy())
|
||||
})
|
||||
{
|
||||
return Err(CredentialSelectionError::NoEligibleCredential);
|
||||
}
|
||||
// 冷却、排除、停用或当前账号不在本次范围时也只等待根线程恢复或迁移
|
||||
waiting
|
||||
.wait(std::slice::from_ref(binding.account_id()))
|
||||
|
||||
@@ -1005,6 +1005,11 @@ pub(super) fn map_client_error(
|
||||
);
|
||||
}
|
||||
if let Some(diagnostic) = diagnostic {
|
||||
if diagnostic.proxy_hint() == Some(ProxyConnectionHint::Unreachable)
|
||||
&& let Some(proxy) = handshake_account.and_then(ProviderAccount::outbound_proxy)
|
||||
{
|
||||
failure.error = failure.error.with_unreachable_proxy(proxy.dial_endpoint());
|
||||
}
|
||||
failure.error = failure.error.with_diagnostic(diagnostic);
|
||||
}
|
||||
if let Some(raw) = raw_upstream_error {
|
||||
@@ -1095,7 +1100,11 @@ pub(super) fn client_diagnostic(
|
||||
.with_classification(stage, code)
|
||||
.with_io_cause(error);
|
||||
return Some(if uses_account_proxy && error.is_connect() {
|
||||
diagnostic.with_proxy_hint(ProxyConnectionHint::CheckProxy)
|
||||
diagnostic.with_proxy_hint(if http_proxy_dial_failed(error) {
|
||||
ProxyConnectionHint::Unreachable
|
||||
} else {
|
||||
ProxyConnectionHint::CheckProxy
|
||||
})
|
||||
} else {
|
||||
diagnostic
|
||||
});
|
||||
@@ -1774,6 +1783,43 @@ pub(super) fn provider_error(
|
||||
ProviderError::new(kind, send_state)
|
||||
}
|
||||
|
||||
/// 账号代理覆盖所有 HTTP 目的地。只有到代理的 TCP 拨号失败可排除整个入口;
|
||||
/// SOCKS 握手和 CONNECT 隧道返回的错误均不能证明入口不可达
|
||||
fn http_proxy_dial_failed(error: &reqwest::Error) -> bool {
|
||||
use hyper_util::client::legacy::connect::{HttpConnector, proxy::SocksV5};
|
||||
use std::error::Error;
|
||||
use tower_service::Service;
|
||||
|
||||
type TcpError = <HttpConnector as Service<hyper::Uri>>::Error;
|
||||
type SocksError = <SocksV5<HttpConnector> as Service<hyper::Uri>>::Error;
|
||||
let mut source = error.source();
|
||||
while let Some(cause) = source {
|
||||
let dial =
|
||||
cause
|
||||
.downcast_ref::<TcpError>()
|
||||
.or_else(|| match cause.downcast_ref::<SocksError>() {
|
||||
Some(SocksError::Inner(dial)) => Some(dial),
|
||||
_ => None,
|
||||
});
|
||||
if let Some(dial) = dial {
|
||||
return dial
|
||||
.source()
|
||||
.and_then(|inner| inner.downcast_ref::<std::io::Error>())
|
||||
.is_some_and(|io| {
|
||||
matches!(
|
||||
io.kind(),
|
||||
std::io::ErrorKind::ConnectionRefused
|
||||
| std::io::ErrorKind::TimedOut
|
||||
| std::io::ErrorKind::NetworkUnreachable
|
||||
| std::io::ErrorKind::HostUnreachable
|
||||
)
|
||||
});
|
||||
}
|
||||
source = cause.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// 只恢复明确未发送的 HTTP 建连失败;TLS 协议、证书及配置错误不反复尝试同一路径
|
||||
fn transient_http_connect(error: &reqwest::Error) -> bool {
|
||||
if !error.is_connect() || error.is_builder() {
|
||||
|
||||
@@ -320,7 +320,7 @@ async fn dial_account(
|
||||
*proxy_hint = None;
|
||||
let tls = tls::account_proxy_tls_config().map_err(|_| invalid())?;
|
||||
let name = rustls_pki_types::ServerName::try_from(proxy_host).map_err(|_| invalid())?;
|
||||
*proxy_hint = Some(ProxyConnectionHint::Unreachable);
|
||||
*proxy_hint = Some(ProxyConnectionHint::CheckProxy);
|
||||
MaybeTlsStream::Rustls(
|
||||
tokio_rustls::TlsConnector::from(tls)
|
||||
.connect(name, tcp)
|
||||
|
||||
@@ -4329,6 +4329,7 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent
|
||||
for (websocket, proxy_hint) in [
|
||||
(false, None),
|
||||
(true, None),
|
||||
(false, Some(ProxyConnectionHint::Unreachable)),
|
||||
(false, Some(ProxyConnectionHint::CheckProxy)),
|
||||
(true, Some(ProxyConnectionHint::Unreachable)),
|
||||
(true, Some(ProxyConnectionHint::TunnelFailed)),
|
||||
@@ -4337,7 +4338,11 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent
|
||||
create_account(&store, "acct_abrupt_disconnect").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let rejection = if proxy_hint == Some(ProxyConnectionHint::TunnelFailed) {
|
||||
let tunnel_rejected = matches!(
|
||||
proxy_hint,
|
||||
Some(ProxyConnectionHint::TunnelFailed | ProxyConnectionHint::CheckProxy)
|
||||
);
|
||||
let rejection = if tunnel_rejected {
|
||||
Some(tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
assert!(
|
||||
@@ -4373,7 +4378,13 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
// HTTP 客户端仅为 HTTPS 目标执行 CONNECT;代理可达但拒绝隧道不能归因为拨号失败
|
||||
let upstream = if !websocket && tunnel_rejected {
|
||||
"https://upstream.invalid".to_owned()
|
||||
} else {
|
||||
base_url
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, upstream)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_connect_refused", CancellationToken::new()),
|
||||
@@ -4389,6 +4400,10 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent
|
||||
};
|
||||
let diagnostic = error.diagnostic().expect("connection diagnosis");
|
||||
assert_eq!(diagnostic.proxy_hint(), proxy_hint);
|
||||
assert_eq!(
|
||||
error.unreachable_proxy().is_some(),
|
||||
proxy_hint == Some(ProxyConnectionHint::Unreachable)
|
||||
);
|
||||
assert_eq!(
|
||||
error.allows_connection_rotation(),
|
||||
proxy_hint != Some(ProxyConnectionHint::TunnelFailed)
|
||||
|
||||
@@ -432,7 +432,10 @@ async fn streaming_not_found_preserves_accounts_and_stops_gateway_retries() {
|
||||
async fn refused_connections_respect_the_same_rotation_limit_over_http_and_websocket() {
|
||||
let upstream = MockServer::start().await;
|
||||
for websocket in [false, true] {
|
||||
for rotations in [0, 2] {
|
||||
for (rotations, child) in [(0, false), (2, false), (2, true)] {
|
||||
if child && websocket {
|
||||
continue;
|
||||
}
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let endpoint = listener.local_addr().unwrap();
|
||||
@@ -443,8 +446,20 @@ async fn refused_connections_respect_the_same_rotation_limit_over_http_and_webso
|
||||
for id in ["acct_provider_contract", "acct_subagent_b"] {
|
||||
create_account(&store, id).await;
|
||||
store.set_egress(id, Some(proxy.clone()), None);
|
||||
if child {
|
||||
store.set_oauth_transport(
|
||||
id,
|
||||
provider_openai::credential::ResponsesTransport::Http,
|
||||
);
|
||||
}
|
||||
}
|
||||
let operation = if websocket {
|
||||
let operation = if child {
|
||||
Operation::Generate(generate_with_session_context(
|
||||
"root",
|
||||
Some("child"),
|
||||
Some(r#"{"subagent_kind":"thread_spawn"}"#),
|
||||
))
|
||||
} else if websocket {
|
||||
capacity_websocket_operation()
|
||||
} else {
|
||||
http_generate_operation()
|
||||
@@ -485,24 +500,127 @@ async fn refused_connections_respect_the_same_rotation_limit_over_http_and_webso
|
||||
);
|
||||
let requests = leases.requests.lock().unwrap();
|
||||
let accounts: Vec<_> = requests.iter().map(|r| r.account_id()).collect();
|
||||
assert!(
|
||||
(3..=4).contains(&accounts.len()),
|
||||
"ws={websocket}, rotations={rotations}: {accounts:?}, {result:?}"
|
||||
);
|
||||
assert_eq!(accounts[0], accounts[1]);
|
||||
if rotations == 0 {
|
||||
assert!(
|
||||
accounts.iter().all(|a| *a == accounts[0]),
|
||||
"ws={websocket}: {accounts:?}"
|
||||
);
|
||||
assert!((3..=4).contains(&accounts.len()), "{accounts:?}");
|
||||
} else {
|
||||
assert_ne!(accounts[1], accounts[2], "ws={websocket}: {accounts:?}");
|
||||
assert!(
|
||||
matches!(&result, Err(EngineError::Provider(error)) if error.unreachable_proxy().is_some()),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
accounts.len(),
|
||||
2,
|
||||
"ws={websocket}: {accounts:?}, {result:?}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
accounts.iter().all(|a| *a == accounts[0]),
|
||||
"ws={websocket}: {accounts:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unreachable_proxy_is_skipped_across_rotations_and_reconsidered_on_the_next_request() {
|
||||
use gateway_core::account::OutboundProxy;
|
||||
|
||||
const A: &str = "acct_provider_contract";
|
||||
const B: &str = "acct_subagent_a";
|
||||
const C: &str = "acct_subagent_b";
|
||||
const D: &str = "acct_affinity";
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let upstream = MockServer::start().await;
|
||||
// 关闭监听以产生真实的连接拒绝,下一请求让同一入口恢复服务
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
for (id, weight) in [(A, 100), (B, 90), (C, 80), (D, 70)] {
|
||||
create_account(&store, id).await;
|
||||
store.set_scheduling(id, None, AccountWeight::new(weight).unwrap());
|
||||
if matches!(id, A | B) {
|
||||
let proxy = OutboundProxy::parse(&format!("http://{id}@{addr}")).unwrap();
|
||||
store.set_egress(id, Some(proxy), None);
|
||||
}
|
||||
}
|
||||
let rate_limit_then_success = || {
|
||||
let calls = std::sync::atomic::AtomicUsize::new(0);
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(move |_: &wiremock::Request| {
|
||||
if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
|
||||
ResponseTemplate::new(429).set_body_json(json!({
|
||||
"error":{"type":"rate_limit_error","message":"fixture rate limit"}
|
||||
}))
|
||||
} else {
|
||||
ResponseTemplate::new(200)
|
||||
.set_body_raw(CAPTURE_COMPLETED_SSE, "text/event-stream")
|
||||
}
|
||||
})
|
||||
.expect(2)
|
||||
};
|
||||
rate_limit_then_success().mount(&upstream).await;
|
||||
let leases = Arc::new(TestLeaseCoordinator::default());
|
||||
let provider = provider_with_affinity_and_base_url_and_leases(
|
||||
&store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
upstream.uri(),
|
||||
Arc::clone(&leases),
|
||||
);
|
||||
let providers =
|
||||
ProviderRegistry::new([provider as Arc<dyn gateway_core::engine::provider::Provider>])
|
||||
.unwrap();
|
||||
let coordinator =
|
||||
AttemptCoordinator::new(GatewayEngine::new(Arc::new(NoopExecutionStore), providers));
|
||||
let run = async |id: &'static str| {
|
||||
// 每个请求启动独立会话,避免已有的会话亲和优先于测试设置的账号权重
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".into(), json!("gpt-5.4")),
|
||||
("input".into(), json!("hello")),
|
||||
("prompt_cache_key".into(), json!(id)),
|
||||
]),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([("use_websocket".into(), json!(false))])),
|
||||
));
|
||||
let plan = execution_plan_with_settings(
|
||||
&operation,
|
||||
SettingsValues::new(3, "smart", Default::default(), None, None)
|
||||
.with_max_account_rotations(2),
|
||||
);
|
||||
let mut session = coordinator
|
||||
.start(
|
||||
execution_request(id, &operation, Duration::from_secs(15)),
|
||||
operation,
|
||||
plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = session.collect_uncommitted().await;
|
||||
let accounts = std::mem::take(&mut *leases.requests.lock().unwrap())
|
||||
.iter()
|
||||
.map(|r| r.account_id().as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(result.is_ok(), "{id}: {result:?}, accounts={accounts:?}");
|
||||
accounts
|
||||
};
|
||||
// C 的限流再次触发换号,B 权重更高也不能换回已失败的入口
|
||||
assert_eq!(run("req_proxy_skip").await, [A, A, C, D]);
|
||||
let recovered = MockServer::builder()
|
||||
.listener(std::net::TcpListener::bind(addr).unwrap())
|
||||
.start()
|
||||
.await;
|
||||
rate_limit_then_success().mount(&recovered).await;
|
||||
// 新请求可重新使用 A;这次 A 只被限流,同一入口的 B 仍是最优账号
|
||||
assert_eq!(run("req_proxy_recovered").await, [A, B]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stalled_http_proxy_retries_same_account_before_using_reserved_rotation_budget() {
|
||||
stalled_proxy_recovery(false).await;
|
||||
|
||||
@@ -176,6 +176,9 @@ impl GrokAccountSessionSelector {
|
||||
.into_iter()
|
||||
.filter(|account| {
|
||||
request.upstream_context.as_ref().is_none_or(|context| {
|
||||
if context.excludes_proxy(account.outbound_proxy()) {
|
||||
return false;
|
||||
}
|
||||
context
|
||||
.upstream_adapter_for_account(
|
||||
&self.provider_kind,
|
||||
|
||||
+3
-2
@@ -772,7 +772,8 @@ xAI 的 `reasoning.effort` 接受 `none / minimal / low / medium / high / xhigh
|
||||
OpenAI HTTP 与 WebSocket 在单请求内共享最多 4 次建连、30 秒预算。
|
||||
HTTP 瞬时建连失败(含 TLS 握手 EOF),或 WebSocket 新请求在发送前遇到连接拒绝、网络不可达等瞬时建连失败时,使用同一换号规则:
|
||||
先在原账号上退避重试;允许换号且至少还能建连两次时,这次重试最多使用一半剩余时间,给重选账号留一次建连机会。
|
||||
再次建连失败才按调度规则重选账号,选号不可用时使用原账号的剩余预算;固定账号或换号预算不足时仅在原账号恢复。
|
||||
再次建连失败才按调度规则重选账号。如果确认连不上账号绑定的代理入口,本请求后续选号会跳过绑定同一主机和端口的账号;没有其他可用账号时返回原错误。新请求重新按正常规则选号。
|
||||
其他瞬时建连失败在没有替代账号时仍可使用原账号的剩余预算;固定账号或换号预算不足时仅在原账号恢复。
|
||||
证书、TLS 协议及配置错误不进入该恢复,取消与请求截止时间优先终止等待
|
||||
|
||||
客户端推理请求提供有效逻辑线程、轮次及 `request_kind=turn` 时,网关按 Client Key 隔离短期失败状态。
|
||||
@@ -2336,7 +2337,7 @@ Chat Completions 和 Responses 通过原生 OpenAI Provider 选号时,缺少
|
||||
|
||||
`maxAccountRotations` 是单请求最大换号次数,默认 3,取值 0~31;0 表示不换号。
|
||||
首次选号、同账号重试和选号时过滤不可用候选不计入换号次数,总路由尝试仍最多 32 次。
|
||||
例如设为 2,A 重试后换到 B 只计 1 次,即使 A、B 使用同一代理。实际换号次数仍受可用账号、建连预算和安全重放条件限制。
|
||||
例如设为 2,A 遇到限流后换到 B 只计 1 次,A、B 可以使用同一代理。实际换号次数仍受可用账号、建连预算和安全重放条件限制。
|
||||
设置更新请求须包含账号亲和、亲和时长与最大换号次数,保存后对新请求生效,执行中请求及其重试沿用冻结值
|
||||
|
||||
### 请求解压上限
|
||||
|
||||
@@ -437,7 +437,9 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别客户端,Cor
|
||||
其他上游拒绝保留响应与重试提示,交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围,明确的 `426` 仍允许 HTTP 回退
|
||||
- `not_sent`、`sent`、`ambiguous` 是单调的上游发送边界;结果不明确时不能假定上游未收到请求
|
||||
- OpenAI HTTP 和无续接依赖的 WS 新请求遇到明确未发送的瞬时建连失败时,先在原账号恢复;
|
||||
Core 按请求级建连与换号预算为重选保留时间和连接次数,再次建连失败才排除原账号;选号不可用时回原账号使用剩余预算。
|
||||
Core 按请求级建连与换号预算为重选保留时间和连接次数,再次建连失败才排除原账号。
|
||||
若 Provider 确认账号绑定的代理入口不可达,Core 仅在决定换号后记录其主机与端口;后续选号排除相同入口,不写入账号健康或共享调度缓存。
|
||||
没有替代账号时交回原错误;未确认入口不可达时仍可回原账号使用剩余预算。
|
||||
Provider 决定恢复传输:HTTP 继续使用 HTTP,WS 按协议重试或回退;
|
||||
外部指定账号或换号预算不足时保留同账号恢复,账号与容量拒绝仍按各自的安全重放规则判断换号;
|
||||
证书、TLS 协议及配置错误不触发这项建连换号策略,取消和执行 deadline 优先终止等待
|
||||
|
||||
Reference in New Issue
Block a user