fix: correct first token timing

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-08-07 18:56:13 +08:00
co-authored by Codex
parent a4a997e351
commit 43604e36dc
7 changed files with 157 additions and 9 deletions
@@ -273,6 +273,8 @@ impl Provider for CodexProvider {
.await
.map_err(map_selection_error)?;
let lease = Arc::new(lease);
// 首字计时的起点:账号选择完成之后、上游建立之前。
let output_started_at = Instant::now();
let provider_kind = ProviderKind::new(PROVIDER_NAME)
.map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent))?;
let cross_account = context
@@ -373,6 +375,7 @@ impl Provider for CodexProvider {
catalog: Arc::clone(&self.catalog),
agent_identity: Arc::clone(&self.agent_identity),
lease: Arc::clone(&lease),
output_started_at,
session_affinity_key,
session_capture,
});
@@ -397,6 +400,7 @@ struct ColdResponse {
catalog: Arc<CodexCredentialCatalogService>,
agent_identity: Arc<CodexAgentIdentityTaskService>,
lease: Arc<CodexCredentialLease>,
output_started_at: Instant,
session_affinity_key: Option<ProviderSessionAffinityKey>,
session_capture: Option<OpenAiSessionCapture>,
}
@@ -555,6 +559,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
catalog,
agent_identity,
lease,
output_started_at,
session_affinity_key,
mut session_capture,
} = response;
@@ -800,7 +805,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
capture.turn_state = Some(turn_state);
}
let first_event_changed =
observation_state.observe_stream_chunk(&chunk, context.timing_started_at());
observation_state.observe_stream_chunk(&chunk, output_started_at);
let chunk_len = chunk.len();
let (mut events, canonical_failure) = match decoder.push(&chunk) {
CodexCanonicalOutcome::Events(events) => (events, None),
@@ -830,7 +835,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
let timing_signals = decoder.take_timing_signals();
let timing_changed = first_event_changed
|| observation_state
.observe_timing_signals(timing_signals, context.timing_started_at());
.observe_timing_signals(timing_signals, output_started_at);
let completed = events
.iter()
.flat_map(ProviderEvent::canonical_facts)
@@ -909,7 +914,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
let service_tier_changed = observation_state
.observe_upstream_service_tier(decoder.response_service_tier());
let timing_changed = observation_state
.observe_timing_signals(timing_signals, context.timing_started_at());
.observe_timing_signals(timing_signals, output_started_at);
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
let rate_limits_changed = if updates.is_empty() {
false
@@ -1063,7 +1068,9 @@ impl OpenAiResponseObservationState {
started_at: Instant,
) -> bool {
let mut changed = false;
if signals.semantic_output {
// 首个非前导输出事件(结构帧也算)开启首字计时;
// 真实语义首字由 first_reasoning_ms / first_text_ms 单独观测。
if signals.output_start {
changed |= insert_first_timing(&mut self.timings.first_token_ms, started_at);
}
if signals.reasoning_output {
@@ -310,6 +310,7 @@ impl CodexCanonicalDecoder {
fn merge_timing_signals(&mut self, signals: ResponseEventSignals) {
self.timing_signals.protocol_progress |= signals.protocol_progress;
self.timing_signals.output_start |= signals.output_start;
self.timing_signals.semantic_output |= signals.semantic_output;
self.timing_signals.reasoning_output |= signals.reasoning_output;
self.timing_signals.text_output |= signals.text_output;
@@ -182,11 +182,13 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
/// 单个 Responses 事件对计时系统提供的稳定语义信号。
///
/// `protocol_progress` 只说明上游仍在工作,不能替代首字;其余字段分别标记
/// 客户端可消费的输出、reasoning 输出与正文输出。
/// `protocol_progress` 只说明上游仍在工作,不能替代首字;`output_start` 标记首个会
/// 开启客户端输出的非前导事件(结构帧也算);其余字段分别标记客户端可消费的
/// 输出、reasoning 输出与正文输出。
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ResponseEventSignals {
pub protocol_progress: bool,
pub output_start: bool,
pub semantic_output: bool,
pub reasoning_output: bool,
pub text_output: bool,
@@ -194,13 +196,22 @@ pub struct ResponseEventSignals {
/// 从已解析的 Responses 事件提取计时语义。
///
/// 生命周期帧和结构帧不会被当作输出。`response.output_item.done` 与终态帧只有
/// 实际携带文本、工具参数、推理或图片结果时才算语义输出。
/// `output_start` 在任意非前导、非失败事件上置位(含结构帧
/// `response.output_item.added`/`content_part.added`),用于开启首字计时;前导帧
/// (`response.created`/`response.in_progress`)与失败帧不置位。语义输出仍要求
/// 实际携带文本、工具参数、推理或图片结果,`response.output_item.done` 与终态帧
/// 只有携带语义内容才算。
pub fn response_event_signals(event_type: Option<&str>, value: &Value) -> ResponseEventSignals {
let mut signals = ResponseEventSignals {
protocol_progress: !matches!(event_type, Some("response.failed" | "error")),
..ResponseEventSignals::default()
};
signals.output_start = event_type.is_some_and(|event_type| {
!matches!(
event_type,
"response.created" | "response.in_progress" | "response.failed" | "error"
)
});
match event_type {
Some("response.output_text.delta") => {
signals.text_output = non_empty_string(value.get("delta"));
@@ -31,6 +31,30 @@ fn response_body_has_semantic_output(body_bytes: &[u8]) -> bool {
})
}
fn response_body_has_output_start(body_bytes: &[u8]) -> bool {
let body = String::from_utf8_lossy(body_bytes);
let lf_end = body.rfind("\n\n").map(|index| index + 2);
let crlf_end = body.rfind("\r\n\r\n").map(|index| index + 4);
let Some(end) = lf_end.into_iter().chain(crlf_end).max() else {
return false;
};
let Ok(events) = parse_sse_events(&body[..end]) else {
return false;
};
events.iter().any(|event| {
let Ok(value) = serde_json::from_str::<Value>(&event.data) else {
return false;
};
let event_type = event.event.as_deref().or_else(|| {
value
.get("type")
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
});
response_event_signals(event_type, &value).output_start
})
}
#[test]
fn semantic_output_should_detect_a_text_delta() {
let body = b"event: response.output_text.delta\ndata: {\"delta\":\"hello\"}\n\n";
@@ -68,6 +92,37 @@ fn semantic_output_should_ignore_response_created() {
assert!(!response_body_has_semantic_output(body));
}
#[test]
fn output_start_should_ignore_preamble_and_failure_frames() {
for body in [
b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{}}\n\n".as_slice(),
b"event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{}}\n\n",
b"event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\"}}\n\n",
b"data: {\"type\":\"error\"}\n\n",
] {
assert!(
!response_body_has_output_start(body),
"frame must not start output: {body:?}"
);
}
}
#[test]
fn output_start_should_count_structural_and_semantic_frames() {
for body in [
// 结构帧无内容也算。
b"event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\"}}\n\n".as_slice(),
b"event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"part\":{\"type\":\"output_text\"}}\n\n",
b"event: response.output_text.delta\ndata: {\"delta\":\"hello\"}\n\n",
b"event: response.reasoning_text.delta\ndata: {\"delta\":\"thinking\"}\n\n",
] {
assert!(
response_body_has_output_start(body),
"frame must start output: {body:?}"
);
}
}
#[test]
fn response_signals_should_not_count_structural_output_item_added() {
let signals = response_event_signals(
+37 -1
View File
@@ -214,6 +214,8 @@ impl GrokBuildProvider {
upstream_request.affinity().cloned(),
)
.await?;
// 首字计时的起点:账号选择完成之后、上游建立之前。
let output_started_at = Instant::now();
apply_continuation(
&mut upstream_request,
previous_session.as_ref(),
@@ -279,6 +281,7 @@ impl GrokBuildProvider {
upstream_model: candidate.upstream_model().clone(),
context,
session: Arc::clone(&selected),
output_started_at,
session_capture,
reasoning_replay_capture,
},
@@ -912,6 +915,7 @@ struct GrokStreamAttempt {
upstream_model: UpstreamModelId,
context: AttemptContext,
session: Arc<SelectedGrokSession>,
output_started_at: Instant,
session_capture: Option<GrokSessionCapture>,
reasoning_replay_capture: Option<GrokReasoningReplayCapture>,
}
@@ -1286,6 +1290,7 @@ fn cold_http_sse_stream(
upstream_model,
context,
session,
output_started_at,
mut session_capture,
mut reasoning_replay_capture,
} = attempt;
@@ -1357,7 +1362,9 @@ fn cold_http_sse_stream(
};
let observation = xai_response_observation(&response)?;
yield ProviderEvent::observation(observation);
let base_timings = observation.timings();
let mut first_token_ms: Option<u64> = None;
yield ProviderEvent::observation(observation.clone());
let mut body = response.into_body();
let mut decoder = GrokCanonicalDecoder::for_request(upstream_model.as_str(), &request);
@@ -1408,6 +1415,21 @@ fn cold_http_sse_stream(
return;
}
};
// 首个非前导输出事件(结构帧也算)即上报携带 first_token_ms 的观测,
// 供 Core 覆盖会话级兜底值。
if decoder.take_output_start() && first_token_ms.is_none() {
first_token_ms = Some(
u64::try_from(output_started_at.elapsed().as_millis()).unwrap_or(u64::MAX),
);
yield ProviderEvent::observation(
observation
.clone()
.with_timings(ProviderResponseTimings {
first_token_ms,
..base_timings
}),
);
}
let completed = events
.iter()
.flat_map(ProviderEvent::canonical_facts)
@@ -1446,6 +1468,20 @@ fn cold_http_sse_stream(
.iter()
.flat_map(ProviderEvent::canonical_facts)
.any(|event| matches!(event, GatewayEvent::Completed(_)));
// 尾部 finish 补全缓冲中的首个输出帧时,同样上报首字。
if decoder.take_output_start() && first_token_ms.is_none() {
first_token_ms = Some(
u64::try_from(output_started_at.elapsed().as_millis()).unwrap_or(u64::MAX),
);
yield ProviderEvent::observation(
observation
.clone()
.with_timings(ProviderResponseTimings {
first_token_ms,
..base_timings
}),
);
}
attach_xai_session_update(&mut final_events, &mut session_capture)?;
if let Some(capture) = reasoning_replay_capture.as_mut() {
capture.observe(&final_events);
@@ -142,6 +142,7 @@ pub struct GrokCanonicalDecoder {
content: BTreeMap<u32, ContentKind>,
tool_arguments_seen: BTreeSet<u32>,
usage_emitted: bool,
output_start_seen: bool,
}
impl GrokCanonicalDecoder {
@@ -156,6 +157,7 @@ impl GrokCanonicalDecoder {
content: BTreeMap::new(),
tool_arguments_seen: BTreeSet::new(),
usage_emitted: false,
output_start_seen: false,
}
}
@@ -188,6 +190,11 @@ impl GrokCanonicalDecoder {
self.decode(events)
}
/// 取走本批解码帧里是否已出现首个非前导输出事件(结构帧也算),用于首字计时。
pub fn take_output_start(&mut self) -> bool {
std::mem::take(&mut self.output_start_seen)
}
fn decode(&mut self, events: Vec<SseEvent>) -> Result<Vec<ProviderEvent>, ProviderError> {
let mut output = Vec::new();
for event in events {
@@ -226,6 +233,11 @@ impl GrokCanonicalDecoder {
if !client_visible_event(&transformed_type) {
continue;
}
// 首个非前导、非失败事件(结构帧也算)开启首字计时。
self.output_start_seen |= !matches!(
transformed_type.as_str(),
"response.created" | "response.in_progress" | "response.failed" | "error"
);
let value = transformed.into_value();
let mut canonical = Vec::new();
// 终态事件(completed/incomplete)fail-closed:用量/计费校验失败即断流。
@@ -114,6 +114,32 @@ fn decoder_should_normalize_text_usage_and_completion() {
));
}
#[test]
fn decoder_output_start_should_ignore_preamble_frames() {
let mut decoder = GrokCanonicalDecoder::new("fallback");
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_os\",\"model\":\"grok-test\"}}\n\n",
"event: response.in_progress\n",
"data: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_os\"}}\n\n",
);
let _ = decoder.push(body.as_bytes()).expect("preamble frames");
assert!(!decoder.take_output_start());
}
#[test]
fn decoder_output_start_should_count_structural_output_item_added() {
let mut decoder = GrokCanonicalDecoder::new("fallback");
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_os\",\"model\":\"grok-test\"}}\n\n",
"event: response.output_item.added\n",
"data: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\"}}\n\n",
);
let _ = decoder.push(body.as_bytes()).expect("output start frame");
assert!(decoder.take_output_start());
}
#[test]
fn decoder_should_emit_each_complete_text_event_without_waiting_for_the_stream_end() {
let mut decoder = GrokCanonicalDecoder::new("grok-4.5");