mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-20 16:01:55 +00:00
fix(gateway): link successful replays to incomplete continuations
This commit is contained in:
@@ -1038,6 +1038,9 @@ impl ProviderError {
|
||||
|
||||
/// 绑定迁移完成后仅替换客户端恢复投影,保留原始失败的诊断与计费事实
|
||||
pub(crate) fn with_client_recovery_from(mut self, recovery: Self) -> Self {
|
||||
if let Some(reason) = recovery.continuation_unavailable_reason() {
|
||||
self = self.with_continuation_unavailable_reason(reason);
|
||||
}
|
||||
self.client_visible_upstream_error = recovery.client_visible_upstream_error;
|
||||
self.client_visible_upstream_response = recovery.client_visible_upstream_response;
|
||||
self.continuation_failure = recovery.continuation_failure;
|
||||
|
||||
@@ -2493,11 +2493,12 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery()
|
||||
|
||||
#[test]
|
||||
fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
for (seconds, replacement, rotations, expected_attempts) in [
|
||||
(309, true, 1, 2),
|
||||
(309, false, 1, 2),
|
||||
(240, true, 1, 1),
|
||||
(309, true, 0, 1),
|
||||
for (seconds, replacement, rotations, committed_prelude, expected_attempts) in [
|
||||
(309, true, 1, false, 2),
|
||||
(317, true, 1, true, 2),
|
||||
(309, false, 1, false, 2),
|
||||
(240, true, 1, false, 1),
|
||||
(309, true, 0, false, 1),
|
||||
] {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
unreachable!()
|
||||
@@ -2516,11 +2517,14 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
Default::default(),
|
||||
240,
|
||||
);
|
||||
let original_body = Bytes::from_static(br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"309","retry-after-ms":"309000"}}}"#);
|
||||
let original_body = json!({"error": {
|
||||
"code": "server_is_overloaded",
|
||||
"headers": {"retry-after": seconds.to_string(), "retry-after-ms": (seconds * 1000).to_string()}
|
||||
}}).to_string();
|
||||
let original = transient_rejection()
|
||||
.with_retry_after(Duration::from_secs(seconds))
|
||||
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
|
||||
String::from_utf8(original_body.to_vec()).unwrap(),
|
||||
original_body.clone(),
|
||||
))
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
@@ -2530,6 +2534,7 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_unavailable_reason("account_rotated")
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
)
|
||||
@@ -2544,10 +2549,18 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
};
|
||||
let mut items = Vec::new();
|
||||
if committed_prelude {
|
||||
items.push(Ok(ProviderEvent::canonical(GatewayEvent::Started(
|
||||
ResponseMeta::new("response-1", "gpt-5"),
|
||||
))
|
||||
.with_replayable_prelude()));
|
||||
}
|
||||
items.push(Err(original));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(original)],
|
||||
items,
|
||||
},
|
||||
Script::Error(recovery),
|
||||
]);
|
||||
@@ -2568,26 +2581,31 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut collection = Box::pin(session.collect_uncommitted());
|
||||
if committed_prelude {
|
||||
block_on(session.next_event()).unwrap().unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
}
|
||||
let mut next = Box::pin(session.next_event());
|
||||
let std::task::Poll::Ready(Err(EngineError::Provider(error))) =
|
||||
block_on(async { futures::poll!(&mut collection) })
|
||||
block_on(async { futures::poll!(&mut next) })
|
||||
else {
|
||||
panic!("native capacity recovery must not wait on the old account");
|
||||
};
|
||||
drop(collection);
|
||||
drop(next);
|
||||
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(seconds)));
|
||||
assert_eq!(
|
||||
error.raw_upstream_error().unwrap().as_str().as_bytes(),
|
||||
original_body.as_ref()
|
||||
);
|
||||
assert_eq!(error.raw_upstream_error().unwrap().as_str(), original_body);
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_error()
|
||||
.and_then(|detail| detail.code()),
|
||||
(replacement && expected_attempts == 2).then_some("previous_response_not_found")
|
||||
);
|
||||
assert_eq!(
|
||||
error.continuation_unavailable_reason(),
|
||||
(replacement && expected_attempts == 2).then_some("account_rotated")
|
||||
);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), expected_attempts);
|
||||
if contexts.len() == 2 {
|
||||
@@ -2602,9 +2620,16 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
|
||||
);
|
||||
assert!(contexts[1].required_account().is_none());
|
||||
}
|
||||
let state = store.state.lock().unwrap();
|
||||
let finalization = &state.finalizations[0];
|
||||
assert_eq!(finalization.attempt_count, 1);
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
1
|
||||
finalization.outcome,
|
||||
if committed_prelude {
|
||||
ExecutionOutcome::Incomplete
|
||||
} else {
|
||||
ExecutionOutcome::Failed
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -674,7 +674,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
and prior.client_api_key_ref = current.client_api_key_ref
|
||||
and prior.continuation_affinity_hash = current.continuation_affinity_hash
|
||||
and prior.continuation_requested
|
||||
and prior.outcome = 'failed'
|
||||
and prior.outcome in ('failed', 'incomplete')
|
||||
and prior.error_kind = 'continuation_recovery_required'
|
||||
and prior.recovered_at is null
|
||||
and current.started_at >= prior.completed_at
|
||||
|
||||
@@ -983,11 +983,12 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
|
||||
failure_completed_at,
|
||||
)
|
||||
.await;
|
||||
seed_recovery_request(
|
||||
seed_continuation_request(
|
||||
&database.pool,
|
||||
"req_continuation_recovery",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(4),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1023,6 +1024,66 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_replay_recovers_only_the_incomplete_request_that_required_it() {
|
||||
let Some(database) = TestDatabase::create("execution_incomplete_replay").await else {
|
||||
return;
|
||||
};
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let affinity_hash = "a".repeat(64);
|
||||
let replay_at = Utc::now() - Duration::seconds(1);
|
||||
for (id, seconds_before_replay, reason) in [
|
||||
("req_rotated", 11, Some("account_rotated")),
|
||||
("req_overloaded", 4, None),
|
||||
] {
|
||||
let completed_at = replay_at - Duration::seconds(seconds_before_replay);
|
||||
let started_at = completed_at - Duration::seconds(1);
|
||||
seed_continuation_request(&database.pool, id, &affinity_hash, started_at, true).await;
|
||||
let mut failed = successful_core_finalization(id);
|
||||
failed.outcome = ExecutionOutcome::Incomplete;
|
||||
failed.downstream_committed_at = Some(started_at.into());
|
||||
failed.completed_at = completed_at.into();
|
||||
failed.error = Some(GatewayError::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"capacity unavailable",
|
||||
));
|
||||
failed.provider_error_code = Some("server_is_overloaded".to_owned());
|
||||
failed.failure_observation.continuation_unavailable_reason = reason.map(str::to_owned);
|
||||
ExecutionStore::finalize_model_request(&store, failed)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
seed_continuation_request(
|
||||
&database.pool,
|
||||
"req_replay",
|
||||
&affinity_hash,
|
||||
replay_at,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
ExecutionStore::finalize_model_request(&store, successful_core_finalization("req_replay"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let requests: Vec<(String, String, String, Option<String>)> = sqlx::query_as(
|
||||
"select id, outcome, provider_error_code, recovery_request_id
|
||||
from model_request_observations where continuation_requested order by id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for (id, outcome, upstream_code, recovered_by) in requests {
|
||||
assert_eq!(outcome, "incomplete");
|
||||
assert_eq!(upstream_code, "server_is_overloaded");
|
||||
assert_eq!(
|
||||
recovered_by.as_deref(),
|
||||
(id == "req_rotated").then_some("req_replay")
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_http_downgrade_should_mark_pending_websocket_failures_recovered() {
|
||||
let Some(database) = TestDatabase::create("execution_transport_downgrade_recovered").await
|
||||
@@ -1220,11 +1281,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved()
|
||||
failure_completed_at,
|
||||
)
|
||||
.await;
|
||||
seed_recovery_request(
|
||||
seed_continuation_request(
|
||||
&database.pool,
|
||||
"req_continuation_retry_failed",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(3),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1250,11 +1312,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved()
|
||||
.expect("load unresolved continuation failure");
|
||||
assert_eq!(after_failed, (None, None, 1));
|
||||
|
||||
seed_recovery_request(
|
||||
seed_continuation_request(
|
||||
&database.pool,
|
||||
"req_continuation_retry_late",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(31),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
ExecutionStore::finalize_model_request(
|
||||
@@ -1310,11 +1373,12 @@ async fn seed_continuation_failure(
|
||||
.expect("finalize seeded continuation failure");
|
||||
}
|
||||
|
||||
async fn seed_recovery_request(
|
||||
async fn seed_continuation_request(
|
||||
pool: &sqlx::PgPool,
|
||||
id: &str,
|
||||
affinity_hash: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
continuation_requested: bool,
|
||||
) {
|
||||
seed_running_request(pool, id)
|
||||
.await
|
||||
@@ -1322,13 +1386,14 @@ async fn seed_recovery_request(
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
|
||||
continuation_affinity_hash = $2, continuation_requested = false,
|
||||
continuation_affinity_hash = $2, continuation_requested = $4,
|
||||
started_at = $3, recovery_lease_expires_at = $3 + interval '1 minute'
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(affinity_hash)
|
||||
.bind(started_at)
|
||||
.bind(continuation_requested)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("prepare seeded recovery request");
|
||||
|
||||
+1
-1
@@ -3046,7 +3046,7 @@ OAuth 字段依据 CPR 导出结构及官方 Codex `TokenData`,不按 `token /
|
||||
`availability.gaps` 明示未记录、淘汰、截断、读取失败与旧记录缺少的决策事实,`omitted` 说明主动省略范围。
|
||||
异步观测尚未落盘、已过期记录、未采集的请求正文与上游内部状态无法由导出补回,`null` 不表示没有发生错误
|
||||
|
||||
错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。
|
||||
错误记录中的“已自动恢复”表示系统关联到了后续成功请求。网关要求重放完整历史时,同一 Key 和会话在 30 秒内开始的新链成功后,会关联到最近一次待恢复的续接。原请求保留“失败”或“未完成”终态及已采集的上游错误。
|
||||
`upstreamSendState = ambiguous` 表示无法确认该次上游执行结果,不代表后续恢复请求失败;
|
||||
恢复关联也不等于逐字节验证过两次请求正文
|
||||
|
||||
|
||||
Reference in New Issue
Block a user