From 43604e36dcc301ffa51ce082aa7d778d4c5f0c29 Mon Sep 17 00:00:00 2001 From: Zyy Date: Fri, 7 Aug 2026 18:53:42 +0800 Subject: [PATCH] fix: correct first token timing Co-authored-by: Codex --- .../crates/providers/openai/src/provider.rs | 15 +++-- .../openai/src/transport/canonical.rs | 1 + .../src/transport/protocol/responses.rs | 19 +++++-- .../tests/transport/protocol/responses.rs | 55 +++++++++++++++++++ backend/crates/providers/xai/src/provider.rs | 38 ++++++++++++- .../providers/xai/src/transport/canonical.rs | 12 ++++ .../xai/tests/transport/canonical.rs | 26 +++++++++ 7 files changed, 157 insertions(+), 9 deletions(-) diff --git a/backend/crates/providers/openai/src/provider.rs b/backend/crates/providers/openai/src/provider.rs index 164285dc2..8228b2c71 100644 --- a/backend/crates/providers/openai/src/provider.rs +++ b/backend/crates/providers/openai/src/provider.rs @@ -273,6 +273,8 @@ impl Provider for CodexProvider { .await .map_err(map_selection_error)?; let lease = Arc::new(lease); + // 首字计时的起点:账号选择完成之后、上游建立之前。 + let output_started_at = Instant::now(); let provider_kind = ProviderKind::new(PROVIDER_NAME) .map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent))?; let cross_account = context @@ -373,6 +375,7 @@ impl Provider for CodexProvider { catalog: Arc::clone(&self.catalog), agent_identity: Arc::clone(&self.agent_identity), lease: Arc::clone(&lease), + output_started_at, session_affinity_key, session_capture, }); @@ -397,6 +400,7 @@ struct ColdResponse { catalog: Arc, agent_identity: Arc, lease: Arc, + output_started_at: Instant, session_affinity_key: Option, session_capture: Option, } @@ -555,6 +559,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream { catalog, agent_identity, lease, + output_started_at, session_affinity_key, mut session_capture, } = response; @@ -800,7 +805,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream { capture.turn_state = Some(turn_state); } let first_event_changed = - observation_state.observe_stream_chunk(&chunk, context.timing_started_at()); + observation_state.observe_stream_chunk(&chunk, output_started_at); let chunk_len = chunk.len(); let (mut events, canonical_failure) = match decoder.push(&chunk) { CodexCanonicalOutcome::Events(events) => (events, None), @@ -830,7 +835,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream { let timing_signals = decoder.take_timing_signals(); let timing_changed = first_event_changed || observation_state - .observe_timing_signals(timing_signals, context.timing_started_at()); + .observe_timing_signals(timing_signals, output_started_at); let completed = events .iter() .flat_map(ProviderEvent::canonical_facts) @@ -909,7 +914,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream { let service_tier_changed = observation_state .observe_upstream_service_tier(decoder.response_service_tier()); let timing_changed = observation_state - .observe_timing_signals(timing_signals, context.timing_started_at()); + .observe_timing_signals(timing_signals, output_started_at); let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await; let rate_limits_changed = if updates.is_empty() { false @@ -1063,7 +1068,9 @@ impl OpenAiResponseObservationState { started_at: Instant, ) -> bool { let mut changed = false; - if signals.semantic_output { + // 首个非前导输出事件(结构帧也算)开启首字计时; + // 真实语义首字由 first_reasoning_ms / first_text_ms 单独观测。 + if signals.output_start { changed |= insert_first_timing(&mut self.timings.first_token_ms, started_at); } if signals.reasoning_output { diff --git a/backend/crates/providers/openai/src/transport/canonical.rs b/backend/crates/providers/openai/src/transport/canonical.rs index 4ffa4eda1..5c7b3c8a8 100644 --- a/backend/crates/providers/openai/src/transport/canonical.rs +++ b/backend/crates/providers/openai/src/transport/canonical.rs @@ -310,6 +310,7 @@ impl CodexCanonicalDecoder { fn merge_timing_signals(&mut self, signals: ResponseEventSignals) { self.timing_signals.protocol_progress |= signals.protocol_progress; + self.timing_signals.output_start |= signals.output_start; self.timing_signals.semantic_output |= signals.semantic_output; self.timing_signals.reasoning_output |= signals.reasoning_output; self.timing_signals.text_output |= signals.text_output; diff --git a/backend/crates/providers/openai/src/transport/protocol/responses.rs b/backend/crates/providers/openai/src/transport/protocol/responses.rs index b88497485..e991fd430 100644 --- a/backend/crates/providers/openai/src/transport/protocol/responses.rs +++ b/backend/crates/providers/openai/src/transport/protocol/responses.rs @@ -182,11 +182,13 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir /// 单个 Responses 事件对计时系统提供的稳定语义信号。 /// -/// `protocol_progress` 只说明上游仍在工作,不能替代首字;其余字段分别标记 -/// 客户端可消费的输出、reasoning 输出与正文输出。 +/// `protocol_progress` 只说明上游仍在工作,不能替代首字;`output_start` 标记首个会 +/// 开启客户端输出的非前导事件(结构帧也算);其余字段分别标记客户端可消费的 +/// 输出、reasoning 输出与正文输出。 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ResponseEventSignals { pub protocol_progress: bool, + pub output_start: bool, pub semantic_output: bool, pub reasoning_output: bool, pub text_output: bool, @@ -194,13 +196,22 @@ pub struct ResponseEventSignals { /// 从已解析的 Responses 事件提取计时语义。 /// -/// 生命周期帧和结构帧不会被当作输出。`response.output_item.done` 与终态帧只有 -/// 实际携带文本、工具参数、推理或图片结果时才算语义输出。 +/// `output_start` 在任意非前导、非失败事件上置位(含结构帧 +/// `response.output_item.added`/`content_part.added`),用于开启首字计时;前导帧 +/// (`response.created`/`response.in_progress`)与失败帧不置位。语义输出仍要求 +/// 实际携带文本、工具参数、推理或图片结果,`response.output_item.done` 与终态帧 +/// 只有携带语义内容才算。 pub fn response_event_signals(event_type: Option<&str>, value: &Value) -> ResponseEventSignals { let mut signals = ResponseEventSignals { protocol_progress: !matches!(event_type, Some("response.failed" | "error")), ..ResponseEventSignals::default() }; + signals.output_start = event_type.is_some_and(|event_type| { + !matches!( + event_type, + "response.created" | "response.in_progress" | "response.failed" | "error" + ) + }); match event_type { Some("response.output_text.delta") => { signals.text_output = non_empty_string(value.get("delta")); diff --git a/backend/crates/providers/openai/tests/transport/protocol/responses.rs b/backend/crates/providers/openai/tests/transport/protocol/responses.rs index 2578cccd5..346611d93 100644 --- a/backend/crates/providers/openai/tests/transport/protocol/responses.rs +++ b/backend/crates/providers/openai/tests/transport/protocol/responses.rs @@ -31,6 +31,30 @@ fn response_body_has_semantic_output(body_bytes: &[u8]) -> bool { }) } +fn response_body_has_output_start(body_bytes: &[u8]) -> bool { + let body = String::from_utf8_lossy(body_bytes); + let lf_end = body.rfind("\n\n").map(|index| index + 2); + let crlf_end = body.rfind("\r\n\r\n").map(|index| index + 4); + let Some(end) = lf_end.into_iter().chain(crlf_end).max() else { + return false; + }; + let Ok(events) = parse_sse_events(&body[..end]) else { + return false; + }; + events.iter().any(|event| { + let Ok(value) = serde_json::from_str::(&event.data) else { + return false; + }; + let event_type = event.event.as_deref().or_else(|| { + value + .get("type") + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + }); + response_event_signals(event_type, &value).output_start + }) +} + #[test] fn semantic_output_should_detect_a_text_delta() { let body = b"event: response.output_text.delta\ndata: {\"delta\":\"hello\"}\n\n"; @@ -68,6 +92,37 @@ fn semantic_output_should_ignore_response_created() { assert!(!response_body_has_semantic_output(body)); } +#[test] +fn output_start_should_ignore_preamble_and_failure_frames() { + for body in [ + b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{}}\n\n".as_slice(), + b"event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{}}\n\n", + b"event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\"}}\n\n", + b"data: {\"type\":\"error\"}\n\n", + ] { + assert!( + !response_body_has_output_start(body), + "frame must not start output: {body:?}" + ); + } +} + +#[test] +fn output_start_should_count_structural_and_semantic_frames() { + for body in [ + // 结构帧无内容也算。 + b"event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\"}}\n\n".as_slice(), + b"event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"part\":{\"type\":\"output_text\"}}\n\n", + b"event: response.output_text.delta\ndata: {\"delta\":\"hello\"}\n\n", + b"event: response.reasoning_text.delta\ndata: {\"delta\":\"thinking\"}\n\n", + ] { + assert!( + response_body_has_output_start(body), + "frame must start output: {body:?}" + ); + } +} + #[test] fn response_signals_should_not_count_structural_output_item_added() { let signals = response_event_signals( diff --git a/backend/crates/providers/xai/src/provider.rs b/backend/crates/providers/xai/src/provider.rs index 4f3615b81..fe9201eb9 100644 --- a/backend/crates/providers/xai/src/provider.rs +++ b/backend/crates/providers/xai/src/provider.rs @@ -214,6 +214,8 @@ impl GrokBuildProvider { upstream_request.affinity().cloned(), ) .await?; + // 首字计时的起点:账号选择完成之后、上游建立之前。 + let output_started_at = Instant::now(); apply_continuation( &mut upstream_request, previous_session.as_ref(), @@ -279,6 +281,7 @@ impl GrokBuildProvider { upstream_model: candidate.upstream_model().clone(), context, session: Arc::clone(&selected), + output_started_at, session_capture, reasoning_replay_capture, }, @@ -912,6 +915,7 @@ struct GrokStreamAttempt { upstream_model: UpstreamModelId, context: AttemptContext, session: Arc, + output_started_at: Instant, session_capture: Option, reasoning_replay_capture: Option, } @@ -1286,6 +1290,7 @@ fn cold_http_sse_stream( upstream_model, context, session, + output_started_at, mut session_capture, mut reasoning_replay_capture, } = attempt; @@ -1357,7 +1362,9 @@ fn cold_http_sse_stream( }; let observation = xai_response_observation(&response)?; - yield ProviderEvent::observation(observation); + let base_timings = observation.timings(); + let mut first_token_ms: Option = None; + yield ProviderEvent::observation(observation.clone()); let mut body = response.into_body(); let mut decoder = GrokCanonicalDecoder::for_request(upstream_model.as_str(), &request); @@ -1408,6 +1415,21 @@ fn cold_http_sse_stream( return; } }; + // 首个非前导输出事件(结构帧也算)即上报携带 first_token_ms 的观测, + // 供 Core 覆盖会话级兜底值。 + if decoder.take_output_start() && first_token_ms.is_none() { + first_token_ms = Some( + u64::try_from(output_started_at.elapsed().as_millis()).unwrap_or(u64::MAX), + ); + yield ProviderEvent::observation( + observation + .clone() + .with_timings(ProviderResponseTimings { + first_token_ms, + ..base_timings + }), + ); + } let completed = events .iter() .flat_map(ProviderEvent::canonical_facts) @@ -1446,6 +1468,20 @@ fn cold_http_sse_stream( .iter() .flat_map(ProviderEvent::canonical_facts) .any(|event| matches!(event, GatewayEvent::Completed(_))); + // 尾部 finish 补全缓冲中的首个输出帧时,同样上报首字。 + if decoder.take_output_start() && first_token_ms.is_none() { + first_token_ms = Some( + u64::try_from(output_started_at.elapsed().as_millis()).unwrap_or(u64::MAX), + ); + yield ProviderEvent::observation( + observation + .clone() + .with_timings(ProviderResponseTimings { + first_token_ms, + ..base_timings + }), + ); + } attach_xai_session_update(&mut final_events, &mut session_capture)?; if let Some(capture) = reasoning_replay_capture.as_mut() { capture.observe(&final_events); diff --git a/backend/crates/providers/xai/src/transport/canonical.rs b/backend/crates/providers/xai/src/transport/canonical.rs index e4545ddb3..e89bbe838 100644 --- a/backend/crates/providers/xai/src/transport/canonical.rs +++ b/backend/crates/providers/xai/src/transport/canonical.rs @@ -142,6 +142,7 @@ pub struct GrokCanonicalDecoder { content: BTreeMap, tool_arguments_seen: BTreeSet, usage_emitted: bool, + output_start_seen: bool, } impl GrokCanonicalDecoder { @@ -156,6 +157,7 @@ impl GrokCanonicalDecoder { content: BTreeMap::new(), tool_arguments_seen: BTreeSet::new(), usage_emitted: false, + output_start_seen: false, } } @@ -188,6 +190,11 @@ impl GrokCanonicalDecoder { self.decode(events) } + /// 取走本批解码帧里是否已出现首个非前导输出事件(结构帧也算),用于首字计时。 + pub fn take_output_start(&mut self) -> bool { + std::mem::take(&mut self.output_start_seen) + } + fn decode(&mut self, events: Vec) -> Result, ProviderError> { let mut output = Vec::new(); for event in events { @@ -226,6 +233,11 @@ impl GrokCanonicalDecoder { if !client_visible_event(&transformed_type) { continue; } + // 首个非前导、非失败事件(结构帧也算)开启首字计时。 + self.output_start_seen |= !matches!( + transformed_type.as_str(), + "response.created" | "response.in_progress" | "response.failed" | "error" + ); let value = transformed.into_value(); let mut canonical = Vec::new(); // 终态事件(completed/incomplete)fail-closed:用量/计费校验失败即断流。 diff --git a/backend/crates/providers/xai/tests/transport/canonical.rs b/backend/crates/providers/xai/tests/transport/canonical.rs index 1376d6f28..db89f4523 100644 --- a/backend/crates/providers/xai/tests/transport/canonical.rs +++ b/backend/crates/providers/xai/tests/transport/canonical.rs @@ -114,6 +114,32 @@ fn decoder_should_normalize_text_usage_and_completion() { )); } +#[test] +fn decoder_output_start_should_ignore_preamble_frames() { + let mut decoder = GrokCanonicalDecoder::new("fallback"); + let body = concat!( + "event: response.created\n", + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_os\",\"model\":\"grok-test\"}}\n\n", + "event: response.in_progress\n", + "data: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_os\"}}\n\n", + ); + let _ = decoder.push(body.as_bytes()).expect("preamble frames"); + assert!(!decoder.take_output_start()); +} + +#[test] +fn decoder_output_start_should_count_structural_output_item_added() { + let mut decoder = GrokCanonicalDecoder::new("fallback"); + let body = concat!( + "event: response.created\n", + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_os\",\"model\":\"grok-test\"}}\n\n", + "event: response.output_item.added\n", + "data: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\"}}\n\n", + ); + let _ = decoder.push(body.as_bytes()).expect("output start frame"); + assert!(decoder.take_output_start()); +} + #[test] fn decoder_should_emit_each_complete_text_event_without_waiting_for_the_stream_end() { let mut decoder = GrokCanonicalDecoder::new("grok-4.5");