mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 00:01:36 +00:00
fix: harden responses websocket recovery
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -31,6 +31,7 @@ pub(crate) enum DispatchFailureClass {
|
||||
CloudflareChallenge,
|
||||
CloudflarePathBlocked,
|
||||
ModelUnsupported,
|
||||
HistoryUnavailable,
|
||||
Upstream,
|
||||
InvalidSse,
|
||||
MissingCompleted,
|
||||
@@ -50,6 +51,7 @@ impl DispatchFailureClass {
|
||||
Self::CloudflareChallenge => "cloudflare_challenge",
|
||||
Self::CloudflarePathBlocked => "cloudflare_path_blocked",
|
||||
Self::ModelUnsupported => "model_unsupported",
|
||||
Self::HistoryUnavailable => "history_unavailable",
|
||||
Self::Upstream => "upstream",
|
||||
Self::InvalidSse => "invalid_sse",
|
||||
Self::MissingCompleted => "missing_completed",
|
||||
|
||||
@@ -63,6 +63,8 @@ pub enum ResponseDispatchError {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("previous response context is unavailable: {upstream_error}")]
|
||||
HistoryUnavailable { upstream_error: String },
|
||||
#[error("upstream request failed: {0}")]
|
||||
Upstream(#[from] CodexClientError),
|
||||
#[error("invalid upstream SSE response: {0}")]
|
||||
@@ -127,7 +129,7 @@ impl ResponseDispatchError {
|
||||
| Self::MissingCompleted
|
||||
| Self::EmptyUpstreamResponse => 502,
|
||||
Self::Failed(failure) => stream_failure_http_status(failure),
|
||||
Self::ModelUnsupported { .. } => 400,
|
||||
Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400,
|
||||
Self::Upstream(error) => upstream_error_http_status(error),
|
||||
}
|
||||
}
|
||||
@@ -186,6 +188,13 @@ impl ResponseDispatchError {
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata {
|
||||
failure_class: DispatchFailureClass::HistoryUnavailable,
|
||||
exhausted_count: None,
|
||||
upstream_error: Some(upstream_error.clone()),
|
||||
upstream_status: Some(400),
|
||||
diagnostics: None,
|
||||
},
|
||||
Self::Upstream(error) => DispatchErrorMetadata::upstream(error),
|
||||
Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse),
|
||||
Self::MissingCompleted => {
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::{
|
||||
protocol::{
|
||||
responses::{
|
||||
reconvert_responses_sse_event_tuple_values, response_sse_event_is_terminal,
|
||||
update_first_response_event_ms,
|
||||
update_first_response_output_ms,
|
||||
},
|
||||
sse::{
|
||||
encode_sse_event, parse_sse_events, sse_body_has_done, sse_frame_end,
|
||||
@@ -78,7 +78,7 @@ pub(super) fn spawn_live_response_stream(
|
||||
context.account_pool.release(&context.account_id).await;
|
||||
return;
|
||||
}
|
||||
update_first_response_event_ms(context.started_at, &body_bytes, &mut first_token_ms);
|
||||
update_first_response_output_ms(context.started_at, &body_bytes, &mut first_token_ms);
|
||||
|
||||
loop {
|
||||
let next = tokio::select! {
|
||||
@@ -104,7 +104,7 @@ pub(super) fn spawn_live_response_stream(
|
||||
context.account_pool.release(&context.account_id).await;
|
||||
return;
|
||||
}
|
||||
update_first_response_event_ms(
|
||||
update_first_response_output_ms(
|
||||
context.started_at,
|
||||
&body_bytes,
|
||||
&mut first_token_ms,
|
||||
|
||||
@@ -331,15 +331,24 @@ impl ResponseDispatchService {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_request_history(
|
||||
async fn try_recover_implicit_resume(
|
||||
&self,
|
||||
request: &mut CodexResponsesRequest,
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
) {
|
||||
account_id: &str,
|
||||
evict_reasoning_replay: bool,
|
||||
) -> bool {
|
||||
let Some(snapshot) = implicit_resume.take() else {
|
||||
return false;
|
||||
};
|
||||
if evict_reasoning_replay {
|
||||
self.evict_reasoning_replay(request, account_id).await;
|
||||
}
|
||||
if let Some(previous_response_id) = request.previous_response_id() {
|
||||
self.session_affinity.forget(previous_response_id).await;
|
||||
}
|
||||
restore_request_without_history(request, implicit_resume);
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
true
|
||||
}
|
||||
|
||||
fn strip_history_if_account_changed(
|
||||
@@ -354,7 +363,10 @@ impl ResponseDispatchService {
|
||||
return;
|
||||
}
|
||||
|
||||
restore_request_without_history(request, implicit_resume);
|
||||
let Some(snapshot) = implicit_resume.take() else {
|
||||
return;
|
||||
};
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
}
|
||||
|
||||
async fn apply_cascading_ban_defense(
|
||||
@@ -363,7 +375,6 @@ impl ResponseDispatchService {
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
preferred_account_id: Option<&str>,
|
||||
acquired_account_id: &str,
|
||||
explicit_previous_response_id: Option<&str>,
|
||||
) -> bool {
|
||||
let Some(preferred_account_id) =
|
||||
preferred_account_id.filter(|account_id| *account_id != acquired_account_id)
|
||||
@@ -392,14 +403,17 @@ impl ResponseDispatchService {
|
||||
return false;
|
||||
}
|
||||
|
||||
let response_id_to_forget = explicit_previous_response_id
|
||||
.or(request.previous_response_id())
|
||||
.map(str::to_string);
|
||||
restore_request_without_history(request, implicit_resume);
|
||||
let response_id_to_forget = request.previous_response_id().map(str::to_string);
|
||||
let restored_full_request = if let Some(snapshot) = implicit_resume.take() {
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if let Some(response_id) = response_id_to_forget {
|
||||
self.session_affinity.forget(&response_id).await;
|
||||
}
|
||||
true
|
||||
restored_full_request
|
||||
}
|
||||
|
||||
async fn evict_reasoning_replay(&self, request: &CodexResponsesRequest, account_id: &str) {
|
||||
@@ -436,12 +450,10 @@ fn strip_request_history(request: &mut CodexResponsesRequest) {
|
||||
request.turn_state = None;
|
||||
}
|
||||
|
||||
fn restore_request_without_history(
|
||||
fn restore_full_request_without_history(
|
||||
request: &mut CodexResponsesRequest,
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
snapshot: ImplicitResumeSnapshot,
|
||||
) {
|
||||
if let Some(snapshot) = implicit_resume.take() {
|
||||
snapshot.restore(request);
|
||||
}
|
||||
snapshot.restore(request);
|
||||
strip_request_history(request);
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ impl ResponseDispatchService {
|
||||
let tuple_schema = request.tuple_schema.clone();
|
||||
let image_generation_requested = request.expects_image_generation();
|
||||
let now = Utc::now();
|
||||
let explicit_previous_response_id = request.previous_response_id().map(ToString::to_string);
|
||||
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
||||
let account_affinity = self.account_affinity_for_request(&request, now).await;
|
||||
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
|
||||
@@ -25,7 +24,6 @@ impl ResponseDispatchService {
|
||||
}
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut history_recovery_used = false;
|
||||
let mut next_required_account_id: Option<String> = None;
|
||||
let mut empty_response_retries = 0u8;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
@@ -135,7 +133,6 @@ impl ResponseDispatchService {
|
||||
&mut implicit_resume,
|
||||
account_affinity.preferred_account_id(),
|
||||
&acquired.account.id,
|
||||
explicit_previous_response_id.as_deref(),
|
||||
)
|
||||
.await;
|
||||
Self::strip_history_if_account_changed(
|
||||
@@ -216,14 +213,16 @@ impl ResponseDispatchService {
|
||||
}
|
||||
}
|
||||
if let CollectedResponse::Failed(failure) = &collected_response {
|
||||
if is_history_recovery_sse_failure(failure) && !history_recovery_used {
|
||||
if sse_failure_invalid_reasoning_replay(failure) {
|
||||
self.evict_reasoning_replay(&request, &release_account_id)
|
||||
.await;
|
||||
}
|
||||
self.recover_request_history(&mut request, &mut implicit_resume)
|
||||
.await;
|
||||
history_recovery_used = true;
|
||||
if is_history_recovery_sse_failure(failure)
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
sse_failure_invalid_reasoning_replay(failure),
|
||||
)
|
||||
.await
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
@@ -308,18 +307,6 @@ impl ResponseDispatchService {
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error)
|
||||
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
|
||||
{
|
||||
if client_error_invalid_reasoning_replay(&error) {
|
||||
self.evict_reasoning_replay(&request, &release_account_id)
|
||||
.await;
|
||||
}
|
||||
self.recover_request_history(&mut request, &mut implicit_resume)
|
||||
.await;
|
||||
history_recovery_used = true;
|
||||
next_required_account_id = Some(release_account_id);
|
||||
}
|
||||
Err(error) if is_auth_upstream_error(&error) => {
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
let account_status = auth_failure_account_status(&error);
|
||||
@@ -396,6 +383,20 @@ impl ResponseDispatchService {
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) => {
|
||||
let history_unavailable = is_history_recovery_upstream_error(&error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(&error),
|
||||
)
|
||||
.await
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
|
||||
recorder: &self.recorder,
|
||||
request_id,
|
||||
@@ -412,6 +413,11 @@ impl ResponseDispatchService {
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
if history_unavailable {
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(&error),
|
||||
});
|
||||
}
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
@@ -477,7 +483,13 @@ impl ResponseDispatchService {
|
||||
Ok(body)
|
||||
}
|
||||
CollectedResponse::Failed(failure) => {
|
||||
let error = ResponseDispatchError::Failed(failure);
|
||||
let error = if is_history_recovery_sse_failure(&failure) {
|
||||
ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: sse_failure_error_body(&failure),
|
||||
}
|
||||
} else {
|
||||
ResponseDispatchError::Failed(failure)
|
||||
};
|
||||
self.record_response_dispatch_error(
|
||||
request_id,
|
||||
route,
|
||||
|
||||
@@ -16,7 +16,6 @@ impl ResponseDispatchService {
|
||||
request.set_stream(true);
|
||||
let tuple_schema = request.tuple_schema.clone();
|
||||
let now = Utc::now();
|
||||
let explicit_previous_response_id = request.previous_response_id().map(ToString::to_string);
|
||||
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
||||
let account_affinity = self.account_affinity_for_request(&request, now).await;
|
||||
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
|
||||
@@ -25,7 +24,6 @@ impl ResponseDispatchService {
|
||||
}
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut history_recovery_used = false;
|
||||
let mut next_required_account_id: Option<String> = None;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
@@ -140,7 +138,6 @@ impl ResponseDispatchService {
|
||||
&mut implicit_resume,
|
||||
account_affinity.preferred_account_id(),
|
||||
&acquired.account.id,
|
||||
explicit_previous_response_id.as_deref(),
|
||||
)
|
||||
.await;
|
||||
Self::strip_history_if_account_changed(
|
||||
@@ -190,21 +187,6 @@ impl ResponseDispatchService {
|
||||
.await;
|
||||
let (prefetched, body) = match prefetch_first_sse_chunk(response.body).await {
|
||||
Ok(prefetched) => prefetched,
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
if is_history_recovery_upstream_error(&error)
|
||||
&& !history_recovery_used =>
|
||||
{
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
if client_error_invalid_reasoning_replay(&error) {
|
||||
self.evict_reasoning_replay(&request, &release_account_id)
|
||||
.await;
|
||||
}
|
||||
self.recover_request_history(&mut request, &mut implicit_resume)
|
||||
.await;
|
||||
history_recovery_used = true;
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
if is_rate_limit_upstream_error(&error) =>
|
||||
{
|
||||
@@ -320,6 +302,21 @@ impl ResponseDispatchService {
|
||||
Err(error) => {
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
if let ResponseDispatchError::Upstream(upstream_error) = &error {
|
||||
let history_unavailable =
|
||||
is_history_recovery_upstream_error(upstream_error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(upstream_error),
|
||||
)
|
||||
.await
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
record_response_upstream_error_event(
|
||||
ResponseUpstreamErrorEventRecord {
|
||||
recorder: &self.recorder,
|
||||
@@ -338,6 +335,11 @@ impl ResponseDispatchService {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if history_unavailable {
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(upstream_error),
|
||||
});
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
return_stream_dispatch_error!(
|
||||
@@ -359,15 +361,17 @@ impl ResponseDispatchService {
|
||||
}
|
||||
};
|
||||
if let Some(failure) = first_failure {
|
||||
if is_history_recovery_sse_failure(&failure) && !history_recovery_used {
|
||||
if is_history_recovery_sse_failure(&failure)
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
sse_failure_invalid_reasoning_replay(&failure),
|
||||
)
|
||||
.await
|
||||
{
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
if sse_failure_invalid_reasoning_replay(&failure) {
|
||||
self.evict_reasoning_replay(&request, &release_account_id)
|
||||
.await;
|
||||
}
|
||||
self.recover_request_history(&mut request, &mut implicit_resume)
|
||||
.await;
|
||||
history_recovery_used = true;
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
@@ -440,7 +444,14 @@ impl ResponseDispatchService {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Err(ResponseDispatchError::Failed(failure.clone()));
|
||||
let error = if is_history_recovery_sse_failure(&failure) {
|
||||
ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: sse_failure_error_body(&failure),
|
||||
}
|
||||
} else {
|
||||
ResponseDispatchError::Failed(failure.clone())
|
||||
};
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let context = LiveResponseStreamContext {
|
||||
@@ -495,19 +506,6 @@ impl ResponseDispatchService {
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error)
|
||||
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
|
||||
{
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
if client_error_invalid_reasoning_replay(&error) {
|
||||
self.evict_reasoning_replay(&request, &release_account_id)
|
||||
.await;
|
||||
}
|
||||
self.recover_request_history(&mut request, &mut implicit_resume)
|
||||
.await;
|
||||
history_recovery_used = true;
|
||||
next_required_account_id = Some(release_account_id);
|
||||
}
|
||||
Err(error) if is_auth_upstream_error(&error) => {
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
@@ -578,6 +576,20 @@ impl ResponseDispatchService {
|
||||
}
|
||||
Err(error) => {
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
let history_unavailable = is_history_recovery_upstream_error(&error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(&error),
|
||||
)
|
||||
.await
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
|
||||
recorder: &self.recorder,
|
||||
request_id,
|
||||
@@ -594,6 +606,11 @@ impl ResponseDispatchService {
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
if history_unavailable {
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(&error),
|
||||
});
|
||||
}
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user