fix: harden responses websocket recovery

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-07-10 20:30:40 +08:00
co-authored by Codex
parent ddd6fe160f
commit 1b5bfb37e1
26 changed files with 786 additions and 478 deletions
+2
View File
@@ -31,6 +31,7 @@ pub(crate) enum DispatchFailureClass {
CloudflareChallenge,
CloudflarePathBlocked,
ModelUnsupported,
HistoryUnavailable,
Upstream,
InvalidSse,
MissingCompleted,
@@ -50,6 +51,7 @@ impl DispatchFailureClass {
Self::CloudflareChallenge => "cloudflare_challenge",
Self::CloudflarePathBlocked => "cloudflare_path_blocked",
Self::ModelUnsupported => "model_unsupported",
Self::HistoryUnavailable => "history_unavailable",
Self::Upstream => "upstream",
Self::InvalidSse => "invalid_sse",
Self::MissingCompleted => "missing_completed",
+10 -1
View File
@@ -63,6 +63,8 @@ pub enum ResponseDispatchError {
count: usize,
upstream_error: String,
},
#[error("previous response context is unavailable: {upstream_error}")]
HistoryUnavailable { upstream_error: String },
#[error("upstream request failed: {0}")]
Upstream(#[from] CodexClientError),
#[error("invalid upstream SSE response: {0}")]
@@ -127,7 +129,7 @@ impl ResponseDispatchError {
| Self::MissingCompleted
| Self::EmptyUpstreamResponse => 502,
Self::Failed(failure) => stream_failure_http_status(failure),
Self::ModelUnsupported { .. } => 400,
Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400,
Self::Upstream(error) => upstream_error_http_status(error),
}
}
@@ -186,6 +188,13 @@ impl ResponseDispatchError {
*count,
upstream_error,
),
Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata {
failure_class: DispatchFailureClass::HistoryUnavailable,
exhausted_count: None,
upstream_error: Some(upstream_error.clone()),
upstream_status: Some(400),
diagnostics: None,
},
Self::Upstream(error) => DispatchErrorMetadata::upstream(error),
Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse),
Self::MissingCompleted => {
@@ -21,7 +21,7 @@ use crate::{
protocol::{
responses::{
reconvert_responses_sse_event_tuple_values, response_sse_event_is_terminal,
update_first_response_event_ms,
update_first_response_output_ms,
},
sse::{
encode_sse_event, parse_sse_events, sse_body_has_done, sse_frame_end,
@@ -78,7 +78,7 @@ pub(super) fn spawn_live_response_stream(
context.account_pool.release(&context.account_id).await;
return;
}
update_first_response_event_ms(context.started_at, &body_bytes, &mut first_token_ms);
update_first_response_output_ms(context.started_at, &body_bytes, &mut first_token_ms);
loop {
let next = tokio::select! {
@@ -104,7 +104,7 @@ pub(super) fn spawn_live_response_stream(
context.account_pool.release(&context.account_id).await;
return;
}
update_first_response_event_ms(
update_first_response_output_ms(
context.started_at,
&body_bytes,
&mut first_token_ms,
+27 -15
View File
@@ -331,15 +331,24 @@ impl ResponseDispatchService {
.await
}
async fn recover_request_history(
async fn try_recover_implicit_resume(
&self,
request: &mut CodexResponsesRequest,
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
) {
account_id: &str,
evict_reasoning_replay: bool,
) -> bool {
let Some(snapshot) = implicit_resume.take() else {
return false;
};
if evict_reasoning_replay {
self.evict_reasoning_replay(request, account_id).await;
}
if let Some(previous_response_id) = request.previous_response_id() {
self.session_affinity.forget(previous_response_id).await;
}
restore_request_without_history(request, implicit_resume);
restore_full_request_without_history(request, snapshot);
true
}
fn strip_history_if_account_changed(
@@ -354,7 +363,10 @@ impl ResponseDispatchService {
return;
}
restore_request_without_history(request, implicit_resume);
let Some(snapshot) = implicit_resume.take() else {
return;
};
restore_full_request_without_history(request, snapshot);
}
async fn apply_cascading_ban_defense(
@@ -363,7 +375,6 @@ impl ResponseDispatchService {
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
preferred_account_id: Option<&str>,
acquired_account_id: &str,
explicit_previous_response_id: Option<&str>,
) -> bool {
let Some(preferred_account_id) =
preferred_account_id.filter(|account_id| *account_id != acquired_account_id)
@@ -392,14 +403,17 @@ impl ResponseDispatchService {
return false;
}
let response_id_to_forget = explicit_previous_response_id
.or(request.previous_response_id())
.map(str::to_string);
restore_request_without_history(request, implicit_resume);
let response_id_to_forget = request.previous_response_id().map(str::to_string);
let restored_full_request = if let Some(snapshot) = implicit_resume.take() {
restore_full_request_without_history(request, snapshot);
true
} else {
false
};
if let Some(response_id) = response_id_to_forget {
self.session_affinity.forget(&response_id).await;
}
true
restored_full_request
}
async fn evict_reasoning_replay(&self, request: &CodexResponsesRequest, account_id: &str) {
@@ -436,12 +450,10 @@ fn strip_request_history(request: &mut CodexResponsesRequest) {
request.turn_state = None;
}
fn restore_request_without_history(
fn restore_full_request_without_history(
request: &mut CodexResponsesRequest,
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
snapshot: ImplicitResumeSnapshot,
) {
if let Some(snapshot) = implicit_resume.take() {
snapshot.restore(request);
}
snapshot.restore(request);
strip_request_history(request);
}
@@ -16,7 +16,6 @@ impl ResponseDispatchService {
let tuple_schema = request.tuple_schema.clone();
let image_generation_requested = request.expects_image_generation();
let now = Utc::now();
let explicit_previous_response_id = request.previous_response_id().map(ToString::to_string);
let mut implicit_resume = self.prepare_response_session(&mut request).await;
let account_affinity = self.account_affinity_for_request(&request, now).await;
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
@@ -25,7 +24,6 @@ impl ResponseDispatchService {
}
let mut excluded_account_ids = Vec::new();
let mut exhausted_accounts = AccountExhaustionTracker::default();
let mut history_recovery_used = false;
let mut next_required_account_id: Option<String> = None;
let mut empty_response_retries = 0u8;
let mut quota_verify_attempts = 0usize;
@@ -135,7 +133,6 @@ impl ResponseDispatchService {
&mut implicit_resume,
account_affinity.preferred_account_id(),
&acquired.account.id,
explicit_previous_response_id.as_deref(),
)
.await;
Self::strip_history_if_account_changed(
@@ -216,14 +213,16 @@ impl ResponseDispatchService {
}
}
if let CollectedResponse::Failed(failure) = &collected_response {
if is_history_recovery_sse_failure(failure) && !history_recovery_used {
if sse_failure_invalid_reasoning_replay(failure) {
self.evict_reasoning_replay(&request, &release_account_id)
.await;
}
self.recover_request_history(&mut request, &mut implicit_resume)
.await;
history_recovery_used = true;
if is_history_recovery_sse_failure(failure)
&& self
.try_recover_implicit_resume(
&mut request,
&mut implicit_resume,
&release_account_id,
sse_failure_invalid_reasoning_replay(failure),
)
.await
{
next_required_account_id = Some(release_account_id);
continue;
}
@@ -308,18 +307,6 @@ impl ResponseDispatchService {
.await;
excluded_account_ids.push(release_account_id);
}
Err(error)
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
{
if client_error_invalid_reasoning_replay(&error) {
self.evict_reasoning_replay(&request, &release_account_id)
.await;
}
self.recover_request_history(&mut request, &mut implicit_resume)
.await;
history_recovery_used = true;
next_required_account_id = Some(release_account_id);
}
Err(error) if is_auth_upstream_error(&error) => {
let upstream_error = upstream_error_body(&error);
let account_status = auth_failure_account_status(&error);
@@ -396,6 +383,20 @@ impl ResponseDispatchService {
excluded_account_ids.push(release_account_id);
}
Err(error) => {
let history_unavailable = is_history_recovery_upstream_error(&error);
if history_unavailable
&& self
.try_recover_implicit_resume(
&mut request,
&mut implicit_resume,
&release_account_id,
client_error_invalid_reasoning_replay(&error),
)
.await
{
next_required_account_id = Some(release_account_id);
continue;
}
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
recorder: &self.recorder,
request_id,
@@ -412,6 +413,11 @@ impl ResponseDispatchService {
attempt: Some(&attempt),
})
.await;
if history_unavailable {
return Err(ResponseDispatchError::HistoryUnavailable {
upstream_error: upstream_error_body(&error),
});
}
return Err(ResponseDispatchError::Upstream(error));
}
}
@@ -477,7 +483,13 @@ impl ResponseDispatchService {
Ok(body)
}
CollectedResponse::Failed(failure) => {
let error = ResponseDispatchError::Failed(failure);
let error = if is_history_recovery_sse_failure(&failure) {
ResponseDispatchError::HistoryUnavailable {
upstream_error: sse_failure_error_body(&failure),
}
} else {
ResponseDispatchError::Failed(failure)
};
self.record_response_dispatch_error(
request_id,
route,
@@ -16,7 +16,6 @@ impl ResponseDispatchService {
request.set_stream(true);
let tuple_schema = request.tuple_schema.clone();
let now = Utc::now();
let explicit_previous_response_id = request.previous_response_id().map(ToString::to_string);
let mut implicit_resume = self.prepare_response_session(&mut request).await;
let account_affinity = self.account_affinity_for_request(&request, now).await;
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
@@ -25,7 +24,6 @@ impl ResponseDispatchService {
}
let mut excluded_account_ids = Vec::new();
let mut exhausted_accounts = AccountExhaustionTracker::default();
let mut history_recovery_used = false;
let mut next_required_account_id: Option<String> = None;
let mut quota_verify_attempts = 0usize;
let mut trace = ResponseDispatchTrace::default();
@@ -140,7 +138,6 @@ impl ResponseDispatchService {
&mut implicit_resume,
account_affinity.preferred_account_id(),
&acquired.account.id,
explicit_previous_response_id.as_deref(),
)
.await;
Self::strip_history_if_account_changed(
@@ -190,21 +187,6 @@ impl ResponseDispatchService {
.await;
let (prefetched, body) = match prefetch_first_sse_chunk(response.body).await {
Ok(prefetched) => prefetched,
Err(ResponseDispatchError::Upstream(error))
if is_history_recovery_upstream_error(&error)
&& !history_recovery_used =>
{
self.account_pool.release(&release_account_id).await;
if client_error_invalid_reasoning_replay(&error) {
self.evict_reasoning_replay(&request, &release_account_id)
.await;
}
self.recover_request_history(&mut request, &mut implicit_resume)
.await;
history_recovery_used = true;
next_required_account_id = Some(release_account_id);
continue;
}
Err(ResponseDispatchError::Upstream(error))
if is_rate_limit_upstream_error(&error) =>
{
@@ -320,6 +302,21 @@ impl ResponseDispatchService {
Err(error) => {
self.account_pool.release(&release_account_id).await;
if let ResponseDispatchError::Upstream(upstream_error) = &error {
let history_unavailable =
is_history_recovery_upstream_error(upstream_error);
if history_unavailable
&& self
.try_recover_implicit_resume(
&mut request,
&mut implicit_resume,
&release_account_id,
client_error_invalid_reasoning_replay(upstream_error),
)
.await
{
next_required_account_id = Some(release_account_id);
continue;
}
record_response_upstream_error_event(
ResponseUpstreamErrorEventRecord {
recorder: &self.recorder,
@@ -338,6 +335,11 @@ impl ResponseDispatchService {
},
)
.await;
if history_unavailable {
return Err(ResponseDispatchError::HistoryUnavailable {
upstream_error: upstream_error_body(upstream_error),
});
}
return Err(error);
}
return_stream_dispatch_error!(
@@ -359,15 +361,17 @@ impl ResponseDispatchService {
}
};
if let Some(failure) = first_failure {
if is_history_recovery_sse_failure(&failure) && !history_recovery_used {
if is_history_recovery_sse_failure(&failure)
&& self
.try_recover_implicit_resume(
&mut request,
&mut implicit_resume,
&release_account_id,
sse_failure_invalid_reasoning_replay(&failure),
)
.await
{
self.account_pool.release(&release_account_id).await;
if sse_failure_invalid_reasoning_replay(&failure) {
self.evict_reasoning_replay(&request, &release_account_id)
.await;
}
self.recover_request_history(&mut request, &mut implicit_resume)
.await;
history_recovery_used = true;
next_required_account_id = Some(release_account_id);
continue;
}
@@ -440,7 +444,14 @@ impl ResponseDispatchService {
},
)
.await;
return Err(ResponseDispatchError::Failed(failure.clone()));
let error = if is_history_recovery_sse_failure(&failure) {
ResponseDispatchError::HistoryUnavailable {
upstream_error: sse_failure_error_body(&failure),
}
} else {
ResponseDispatchError::Failed(failure.clone())
};
return Err(error);
}
let context = LiveResponseStreamContext {
@@ -495,19 +506,6 @@ impl ResponseDispatchService {
.await;
excluded_account_ids.push(release_account_id);
}
Err(error)
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
{
self.account_pool.release(&release_account_id).await;
if client_error_invalid_reasoning_replay(&error) {
self.evict_reasoning_replay(&request, &release_account_id)
.await;
}
self.recover_request_history(&mut request, &mut implicit_resume)
.await;
history_recovery_used = true;
next_required_account_id = Some(release_account_id);
}
Err(error) if is_auth_upstream_error(&error) => {
self.account_pool.release(&release_account_id).await;
let upstream_error = upstream_error_body(&error);
@@ -578,6 +576,20 @@ impl ResponseDispatchService {
}
Err(error) => {
self.account_pool.release(&release_account_id).await;
let history_unavailable = is_history_recovery_upstream_error(&error);
if history_unavailable
&& self
.try_recover_implicit_resume(
&mut request,
&mut implicit_resume,
&release_account_id,
client_error_invalid_reasoning_replay(&error),
)
.await
{
next_required_account_id = Some(release_account_id);
continue;
}
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
recorder: &self.recorder,
request_id,
@@ -594,6 +606,11 @@ impl ResponseDispatchService {
attempt: Some(&attempt),
})
.await;
if history_unavailable {
return Err(ResponseDispatchError::HistoryUnavailable {
upstream_error: upstream_error_body(&error),
});
}
return Err(ResponseDispatchError::Upstream(error));
}
}