fix(gateway): link successful replays to incomplete continuations

This commit is contained in:
Zyy
2026-11-20 17:30:24 +08:00
parent 605b60e58f
commit 6fc6283f3a
5 changed files with 118 additions and 25 deletions
@@ -1038,6 +1038,9 @@ impl ProviderError {
/// 绑定迁移完成后仅替换客户端恢复投影,保留原始失败的诊断与计费事实
pub(crate) fn with_client_recovery_from(mut self, recovery: Self) -> Self {
if let Some(reason) = recovery.continuation_unavailable_reason() {
self = self.with_continuation_unavailable_reason(reason);
}
self.client_visible_upstream_error = recovery.client_visible_upstream_error;
self.client_visible_upstream_response = recovery.client_visible_upstream_response;
self.continuation_failure = recovery.continuation_failure;
@@ -2493,11 +2493,12 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery()
#[test]
fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
for (seconds, replacement, rotations, expected_attempts) in [
(309, true, 1, 2),
(309, false, 1, 2),
(240, true, 1, 1),
(309, true, 0, 1),
for (seconds, replacement, rotations, committed_prelude, expected_attempts) in [
(309, true, 1, false, 2),
(317, true, 1, true, 2),
(309, false, 1, false, 2),
(240, true, 1, false, 1),
(309, true, 0, false, 1),
] {
let Operation::Generate(generate) = generate_operation() else {
unreachable!()
@@ -2516,11 +2517,14 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
Default::default(),
240,
);
let original_body = Bytes::from_static(br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"309","retry-after-ms":"309000"}}}"#);
let original_body = json!({"error": {
"code": "server_is_overloaded",
"headers": {"retry-after": seconds.to_string(), "retry-after-ms": (seconds * 1000).to_string()}
}}).to_string();
let original = transient_rejection()
.with_retry_after(Duration::from_secs(seconds))
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
String::from_utf8(original_body.to_vec()).unwrap(),
original_body.clone(),
))
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
@@ -2530,6 +2534,7 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_continuation_unavailable_reason("account_rotated")
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
)
@@ -2544,10 +2549,18 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
UpstreamSendState::NotSent,
)
};
let mut items = Vec::new();
if committed_prelude {
items.push(Ok(ProviderEvent::canonical(GatewayEvent::Started(
ResponseMeta::new("response-1", "gpt-5"),
))
.with_replayable_prelude()));
}
items.push(Err(original));
let (coordinator, store, provider) = coordinator(vec![
Script::Stream {
Script::ObservedStream {
account_id: "acct_one",
items: vec![Err(original)],
items,
},
Script::Error(recovery),
]);
@@ -2568,26 +2581,31 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
CancellationToken::new(),
))
.unwrap();
let mut collection = Box::pin(session.collect_uncommitted());
if committed_prelude {
block_on(session.next_event()).unwrap().unwrap();
block_on(session.commit_downstream(Some(200))).unwrap();
}
let mut next = Box::pin(session.next_event());
let std::task::Poll::Ready(Err(EngineError::Provider(error))) =
block_on(async { futures::poll!(&mut collection) })
block_on(async { futures::poll!(&mut next) })
else {
panic!("native capacity recovery must not wait on the old account");
};
drop(collection);
drop(next);
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
assert_eq!(error.send_state(), UpstreamSendState::Sent);
assert_eq!(error.retry_after(), Some(Duration::from_secs(seconds)));
assert_eq!(
error.raw_upstream_error().unwrap().as_str().as_bytes(),
original_body.as_ref()
);
assert_eq!(error.raw_upstream_error().unwrap().as_str(), original_body);
assert_eq!(
error
.client_visible_upstream_error()
.and_then(|detail| detail.code()),
(replacement && expected_attempts == 2).then_some("previous_response_not_found")
);
assert_eq!(
error.continuation_unavailable_reason(),
(replacement && expected_attempts == 2).then_some("account_rotated")
);
let contexts = provider.contexts.lock().unwrap();
assert_eq!(contexts.len(), expected_attempts);
if contexts.len() == 2 {
@@ -2602,9 +2620,16 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() {
);
assert!(contexts[1].required_account().is_none());
}
let state = store.state.lock().unwrap();
let finalization = &state.finalizations[0];
assert_eq!(finalization.attempt_count, 1);
assert_eq!(
store.state.lock().unwrap().finalizations[0].attempt_count,
1
finalization.outcome,
if committed_prelude {
ExecutionOutcome::Incomplete
} else {
ExecutionOutcome::Failed
}
);
}
}
@@ -674,7 +674,7 @@ impl ModelRequestRepository for PgExecutionStore {
and prior.client_api_key_ref = current.client_api_key_ref
and prior.continuation_affinity_hash = current.continuation_affinity_hash
and prior.continuation_requested
and prior.outcome = 'failed'
and prior.outcome in ('failed', 'incomplete')
and prior.error_kind = 'continuation_recovery_required'
and prior.recovered_at is null
and current.started_at >= prior.completed_at
@@ -983,11 +983,12 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
failure_completed_at,
)
.await;
seed_recovery_request(
seed_continuation_request(
&database.pool,
"req_continuation_recovery",
&affinity_hash,
failure_completed_at + Duration::seconds(4),
false,
)
.await;
@@ -1023,6 +1024,66 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
database.close().await;
}
#[tokio::test]
async fn successful_replay_recovers_only_the_incomplete_request_that_required_it() {
let Some(database) = TestDatabase::create("execution_incomplete_replay").await else {
return;
};
let store = PgExecutionStore::new(database.pool.clone());
let affinity_hash = "a".repeat(64);
let replay_at = Utc::now() - Duration::seconds(1);
for (id, seconds_before_replay, reason) in [
("req_rotated", 11, Some("account_rotated")),
("req_overloaded", 4, None),
] {
let completed_at = replay_at - Duration::seconds(seconds_before_replay);
let started_at = completed_at - Duration::seconds(1);
seed_continuation_request(&database.pool, id, &affinity_hash, started_at, true).await;
let mut failed = successful_core_finalization(id);
failed.outcome = ExecutionOutcome::Incomplete;
failed.downstream_committed_at = Some(started_at.into());
failed.completed_at = completed_at.into();
failed.error = Some(GatewayError::new(
GatewayErrorKind::UpstreamUnavailable,
"capacity unavailable",
));
failed.provider_error_code = Some("server_is_overloaded".to_owned());
failed.failure_observation.continuation_unavailable_reason = reason.map(str::to_owned);
ExecutionStore::finalize_model_request(&store, failed)
.await
.unwrap();
}
seed_continuation_request(
&database.pool,
"req_replay",
&affinity_hash,
replay_at,
false,
)
.await;
ExecutionStore::finalize_model_request(&store, successful_core_finalization("req_replay"))
.await
.unwrap();
let requests: Vec<(String, String, String, Option<String>)> = sqlx::query_as(
"select id, outcome, provider_error_code, recovery_request_id
from model_request_observations where continuation_requested order by id",
)
.fetch_all(&database.pool)
.await
.unwrap();
assert_eq!(requests.len(), 2);
for (id, outcome, upstream_code, recovered_by) in requests {
assert_eq!(outcome, "incomplete");
assert_eq!(upstream_code, "server_is_overloaded");
assert_eq!(
recovered_by.as_deref(),
(id == "req_rotated").then_some("req_replay")
);
}
database.close().await;
}
#[tokio::test]
async fn successful_http_downgrade_should_mark_pending_websocket_failures_recovered() {
let Some(database) = TestDatabase::create("execution_transport_downgrade_recovered").await
@@ -1220,11 +1281,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved()
failure_completed_at,
)
.await;
seed_recovery_request(
seed_continuation_request(
&database.pool,
"req_continuation_retry_failed",
&affinity_hash,
failure_completed_at + Duration::seconds(3),
false,
)
.await;
@@ -1250,11 +1312,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved()
.expect("load unresolved continuation failure");
assert_eq!(after_failed, (None, None, 1));
seed_recovery_request(
seed_continuation_request(
&database.pool,
"req_continuation_retry_late",
&affinity_hash,
failure_completed_at + Duration::seconds(31),
false,
)
.await;
ExecutionStore::finalize_model_request(
@@ -1310,11 +1373,12 @@ async fn seed_continuation_failure(
.expect("finalize seeded continuation failure");
}
async fn seed_recovery_request(
async fn seed_continuation_request(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
started_at: chrono::DateTime<Utc>,
continuation_requested: bool,
) {
seed_running_request(pool, id)
.await
@@ -1322,13 +1386,14 @@ async fn seed_recovery_request(
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
continuation_affinity_hash = $2, continuation_requested = false,
continuation_affinity_hash = $2, continuation_requested = $4,
started_at = $3, recovery_lease_expires_at = $3 + interval '1 minute'
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind(started_at)
.bind(continuation_requested)
.execute(pool)
.await
.expect("prepare seeded recovery request");
+1 -1
View File
@@ -3046,7 +3046,7 @@ OAuth 字段依据 CPR 导出结构及官方 Codex `TokenData`,不按 `token /
`availability.gaps` 明示未记录、淘汰、截断、读取失败与旧记录缺少的决策事实,`omitted` 说明主动省略范围。
异步观测尚未落盘、已过期记录、未采集的请求正文与上游内部状态无法由导出补回,`null` 不表示没有发生错误
错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。
错误记录中的“已自动恢复”表示系统关联到了后续成功请求。网关要求重放完整历史时,同一 Key 和会话在 30 秒内开始的新链成功后,会关联到最近一次待恢复的续接。原请求保留“失败”或“未完成”终态及已采集的上游错误。
`upstreamSendState = ambiguous` 表示无法确认该次上游执行结果,不代表后续恢复请求失败;
恢复关联也不等于逐字节验证过两次请求正文