diff --git a/backend/crates/gateway-core/src/error/mod.rs b/backend/crates/gateway-core/src/error/mod.rs index ac25101d1..981462c29 100644 --- a/backend/crates/gateway-core/src/error/mod.rs +++ b/backend/crates/gateway-core/src/error/mod.rs @@ -1038,6 +1038,9 @@ impl ProviderError { /// 绑定迁移完成后仅替换客户端恢复投影,保留原始失败的诊断与计费事实 pub(crate) fn with_client_recovery_from(mut self, recovery: Self) -> Self { + if let Some(reason) = recovery.continuation_unavailable_reason() { + self = self.with_continuation_unavailable_reason(reason); + } self.client_visible_upstream_error = recovery.client_visible_upstream_error; self.client_visible_upstream_response = recovery.client_visible_upstream_response; self.continuation_failure = recovery.continuation_failure; diff --git a/backend/crates/gateway-core/tests/engine/coordinator.rs b/backend/crates/gateway-core/tests/engine/coordinator.rs index 7f3bcd08e..72e96babd 100644 --- a/backend/crates/gateway-core/tests/engine/coordinator.rs +++ b/backend/crates/gateway-core/tests/engine/coordinator.rs @@ -2493,11 +2493,12 @@ fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery() #[test] fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { - for (seconds, replacement, rotations, expected_attempts) in [ - (309, true, 1, 2), - (309, false, 1, 2), - (240, true, 1, 1), - (309, true, 0, 1), + for (seconds, replacement, rotations, committed_prelude, expected_attempts) in [ + (309, true, 1, false, 2), + (317, true, 1, true, 2), + (309, false, 1, false, 2), + (240, true, 1, false, 1), + (309, true, 0, false, 1), ] { let Operation::Generate(generate) = generate_operation() else { unreachable!() @@ -2516,11 +2517,14 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { Default::default(), 240, ); - let original_body = Bytes::from_static(br#"{"error":{"code":"server_is_overloaded","headers":{"retry-after":"309","retry-after-ms":"309000"}}}"#); + let original_body = json!({"error": { + "code": "server_is_overloaded", + "headers": {"retry-after": seconds.to_string(), "retry-after-ms": (seconds * 1000).to_string()} + }}).to_string(); let original = transient_rejection() .with_retry_after(Duration::from_secs(seconds)) .with_raw_upstream_error(gateway_core::error::RawUpstreamError::new( - String::from_utf8(original_body.to_vec()).unwrap(), + original_body.clone(), )) .with_continuation_recovery_disposition( ContinuationRecoveryDisposition::ClientReplayRequired, @@ -2530,6 +2534,7 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { ProviderErrorKind::ContinuationRecoveryRequired, UpstreamSendState::NotSent, ) + .with_continuation_unavailable_reason("account_rotated") .with_continuation_recovery_disposition( ContinuationRecoveryDisposition::ClientReplayRequired, ) @@ -2544,10 +2549,18 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { UpstreamSendState::NotSent, ) }; + let mut items = Vec::new(); + if committed_prelude { + items.push(Ok(ProviderEvent::canonical(GatewayEvent::Started( + ResponseMeta::new("response-1", "gpt-5"), + )) + .with_replayable_prelude())); + } + items.push(Err(original)); let (coordinator, store, provider) = coordinator(vec![ - Script::Stream { + Script::ObservedStream { account_id: "acct_one", - items: vec![Err(original)], + items, }, Script::Error(recovery), ]); @@ -2568,26 +2581,31 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { CancellationToken::new(), )) .unwrap(); - let mut collection = Box::pin(session.collect_uncommitted()); + if committed_prelude { + block_on(session.next_event()).unwrap().unwrap(); + block_on(session.commit_downstream(Some(200))).unwrap(); + } + let mut next = Box::pin(session.next_event()); let std::task::Poll::Ready(Err(EngineError::Provider(error))) = - block_on(async { futures::poll!(&mut collection) }) + block_on(async { futures::poll!(&mut next) }) else { panic!("native capacity recovery must not wait on the old account"); }; - drop(collection); + drop(next); assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable); assert_eq!(error.send_state(), UpstreamSendState::Sent); assert_eq!(error.retry_after(), Some(Duration::from_secs(seconds))); - assert_eq!( - error.raw_upstream_error().unwrap().as_str().as_bytes(), - original_body.as_ref() - ); + assert_eq!(error.raw_upstream_error().unwrap().as_str(), original_body); assert_eq!( error .client_visible_upstream_error() .and_then(|detail| detail.code()), (replacement && expected_attempts == 2).then_some("previous_response_not_found") ); + assert_eq!( + error.continuation_unavailable_reason(), + (replacement && expected_attempts == 2).then_some("account_rotated") + ); let contexts = provider.contexts.lock().unwrap(); assert_eq!(contexts.len(), expected_attempts); if contexts.len() == 2 { @@ -2602,9 +2620,16 @@ fn native_capacity_above_wait_limit_rotates_before_requesting_client_replay() { ); assert!(contexts[1].required_account().is_none()); } + let state = store.state.lock().unwrap(); + let finalization = &state.finalizations[0]; + assert_eq!(finalization.attempt_count, 1); assert_eq!( - store.state.lock().unwrap().finalizations[0].attempt_count, - 1 + finalization.outcome, + if committed_prelude { + ExecutionOutcome::Incomplete + } else { + ExecutionOutcome::Failed + } ); } } diff --git a/backend/crates/gateway-store/src/postgres/execution/mod.rs b/backend/crates/gateway-store/src/postgres/execution/mod.rs index e44b4f7e0..cd7da5ca6 100644 --- a/backend/crates/gateway-store/src/postgres/execution/mod.rs +++ b/backend/crates/gateway-store/src/postgres/execution/mod.rs @@ -674,7 +674,7 @@ impl ModelRequestRepository for PgExecutionStore { and prior.client_api_key_ref = current.client_api_key_ref and prior.continuation_affinity_hash = current.continuation_affinity_hash and prior.continuation_requested - and prior.outcome = 'failed' + and prior.outcome in ('failed', 'incomplete') and prior.error_kind = 'continuation_recovery_required' and prior.recovered_at is null and current.started_at >= prior.completed_at diff --git a/backend/crates/gateway-store/tests/postgres/execution/mod.rs b/backend/crates/gateway-store/tests/postgres/execution/mod.rs index 7b49b555c..12574e9dc 100644 --- a/backend/crates/gateway-store/tests/postgres/execution/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/execution/mod.rs @@ -983,11 +983,12 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered( failure_completed_at, ) .await; - seed_recovery_request( + seed_continuation_request( &database.pool, "req_continuation_recovery", &affinity_hash, failure_completed_at + Duration::seconds(4), + false, ) .await; @@ -1023,6 +1024,66 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered( database.close().await; } +#[tokio::test] +async fn successful_replay_recovers_only_the_incomplete_request_that_required_it() { + let Some(database) = TestDatabase::create("execution_incomplete_replay").await else { + return; + }; + let store = PgExecutionStore::new(database.pool.clone()); + let affinity_hash = "a".repeat(64); + let replay_at = Utc::now() - Duration::seconds(1); + for (id, seconds_before_replay, reason) in [ + ("req_rotated", 11, Some("account_rotated")), + ("req_overloaded", 4, None), + ] { + let completed_at = replay_at - Duration::seconds(seconds_before_replay); + let started_at = completed_at - Duration::seconds(1); + seed_continuation_request(&database.pool, id, &affinity_hash, started_at, true).await; + let mut failed = successful_core_finalization(id); + failed.outcome = ExecutionOutcome::Incomplete; + failed.downstream_committed_at = Some(started_at.into()); + failed.completed_at = completed_at.into(); + failed.error = Some(GatewayError::new( + GatewayErrorKind::UpstreamUnavailable, + "capacity unavailable", + )); + failed.provider_error_code = Some("server_is_overloaded".to_owned()); + failed.failure_observation.continuation_unavailable_reason = reason.map(str::to_owned); + ExecutionStore::finalize_model_request(&store, failed) + .await + .unwrap(); + } + seed_continuation_request( + &database.pool, + "req_replay", + &affinity_hash, + replay_at, + false, + ) + .await; + ExecutionStore::finalize_model_request(&store, successful_core_finalization("req_replay")) + .await + .unwrap(); + + let requests: Vec<(String, String, String, Option)> = sqlx::query_as( + "select id, outcome, provider_error_code, recovery_request_id + from model_request_observations where continuation_requested order by id", + ) + .fetch_all(&database.pool) + .await + .unwrap(); + assert_eq!(requests.len(), 2); + for (id, outcome, upstream_code, recovered_by) in requests { + assert_eq!(outcome, "incomplete"); + assert_eq!(upstream_code, "server_is_overloaded"); + assert_eq!( + recovered_by.as_deref(), + (id == "req_rotated").then_some("req_replay") + ); + } + database.close().await; +} + #[tokio::test] async fn successful_http_downgrade_should_mark_pending_websocket_failures_recovered() { let Some(database) = TestDatabase::create("execution_transport_downgrade_recovered").await @@ -1220,11 +1281,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() failure_completed_at, ) .await; - seed_recovery_request( + seed_continuation_request( &database.pool, "req_continuation_retry_failed", &affinity_hash, failure_completed_at + Duration::seconds(3), + false, ) .await; @@ -1250,11 +1312,12 @@ async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() .expect("load unresolved continuation failure"); assert_eq!(after_failed, (None, None, 1)); - seed_recovery_request( + seed_continuation_request( &database.pool, "req_continuation_retry_late", &affinity_hash, failure_completed_at + Duration::seconds(31), + false, ) .await; ExecutionStore::finalize_model_request( @@ -1310,11 +1373,12 @@ async fn seed_continuation_failure( .expect("finalize seeded continuation failure"); } -async fn seed_recovery_request( +async fn seed_continuation_request( pool: &sqlx::PgPool, id: &str, affinity_hash: &str, started_at: chrono::DateTime, + continuation_requested: bool, ) { seed_running_request(pool, id) .await @@ -1322,13 +1386,14 @@ async fn seed_recovery_request( sqlx::query( "update model_requests set client_api_key_ref = 'key_continuation', client_transport = 'websocket', - continuation_affinity_hash = $2, continuation_requested = false, + continuation_affinity_hash = $2, continuation_requested = $4, started_at = $3, recovery_lease_expires_at = $3 + interval '1 minute' where id = $1", ) .bind(id) .bind(affinity_hash) .bind(started_at) + .bind(continuation_requested) .execute(pool) .await .expect("prepare seeded recovery request"); diff --git a/docs/api.md b/docs/api.md index 2c16bd553..00779cc62 100644 --- a/docs/api.md +++ b/docs/api.md @@ -3046,7 +3046,7 @@ OAuth 字段依据 CPR 导出结构及官方 Codex `TokenData`,不按 `token / `availability.gaps` 明示未记录、淘汰、截断、读取失败与旧记录缺少的决策事实,`omitted` 说明主动省略范围。 异步观测尚未落盘、已过期记录、未采集的请求正文与上游内部状态无法由导出补回,`null` 不表示没有发生错误 -错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。 +错误记录中的“已自动恢复”表示系统关联到了后续成功请求。网关要求重放完整历史时,同一 Key 和会话在 30 秒内开始的新链成功后,会关联到最近一次待恢复的续接。原请求保留“失败”或“未完成”终态及已采集的上游错误。 `upstreamSendState = ambiguous` 表示无法确认该次上游执行结果,不代表后续恢复请求失败; 恢复关联也不等于逐字节验证过两次请求正文