mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-20 08:01:46 +00:00
Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
1074 lines
38 KiB
Rust
1074 lines
38 KiB
Rust
//! 验证插件观察计划的范围、顺序、去重与有界异步通知
|
|
|
|
use std::{collections::BTreeMap, num::NonZeroU32, sync::Arc, time::Duration};
|
|
|
|
use gateway_admin::{
|
|
model::{
|
|
Revision,
|
|
plugins::instances::{
|
|
PluginCapabilityBinding, PluginFailurePolicy, PluginInstance, PluginInstanceSnapshot,
|
|
},
|
|
},
|
|
ports::plugins::PluginPackageInspector,
|
|
};
|
|
use gateway_core::{
|
|
account::ProviderAccountId,
|
|
engine::{
|
|
ModelRequestId, ModelRequestTimings,
|
|
observation::{
|
|
RequestObservation, RequestObservationOutcome, RequestObservationScope,
|
|
WebSocketResponseAttempt, WebSocketResponseObservation,
|
|
},
|
|
},
|
|
error::GatewayErrorKind,
|
|
event::ProtocolWireEvent,
|
|
identity::ProviderKind,
|
|
metering::{ProviderReportedCost, Usage},
|
|
operation::OperationKind,
|
|
policy::ClientApiKeyId,
|
|
routing::{AccountGroupId, ConfigRevision, PublicModelId},
|
|
upstream::UpstreamSendState,
|
|
};
|
|
use gateway_plugin_runtime::{
|
|
PackageInspector, PackageLimits, PluginRuntime, PluginRuntimeConfig, RpcLimits,
|
|
};
|
|
use gateway_plugin_sdk::{Capability, Contributions, Stage, call::observation::EventKind};
|
|
|
|
use crate::support::store::Store;
|
|
|
|
fn binding(event: EventKind, order: i32) -> PluginCapabilityBinding {
|
|
PluginCapabilityBinding {
|
|
contribution: "test.example.observer".into(),
|
|
stage: "observation".into(),
|
|
order,
|
|
failure_policy: PluginFailurePolicy::Observe,
|
|
client_key_ids: vec![],
|
|
account_group_ids: vec![],
|
|
provider_ids: vec!["openai".into()],
|
|
models: vec!["gpt-observed".into()],
|
|
event: Some(
|
|
serde_json::to_value(event)
|
|
.unwrap()
|
|
.as_str()
|
|
.unwrap()
|
|
.into(),
|
|
),
|
|
identity_bindings: vec![],
|
|
}
|
|
}
|
|
|
|
struct InstanceFixture {
|
|
id: String,
|
|
configuration: serde_json::Value,
|
|
|
|
bindings: Vec<PluginCapabilityBinding>,
|
|
}
|
|
|
|
async fn setup(
|
|
instances: Vec<(&str, serde_json::Value, Vec<PluginCapabilityBinding>)>,
|
|
rpc_limits: RpcLimits,
|
|
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
|
setup_instances(
|
|
instances
|
|
.into_iter()
|
|
.map(|(id, configuration, bindings)| InstanceFixture {
|
|
id: id.to_owned(),
|
|
configuration,
|
|
|
|
bindings,
|
|
})
|
|
.collect(),
|
|
rpc_limits,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn setup_instances(
|
|
instances: Vec<InstanceFixture>,
|
|
|
|
rpc_limits: RpcLimits,
|
|
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
|
let cache = tempfile::tempdir().unwrap();
|
|
let contributes = Contributions::from([crate::support::contribution(
|
|
Capability::Observer,
|
|
vec![Stage::Observation],
|
|
vec![],
|
|
vec![],
|
|
)]);
|
|
let artifact = PackageInspector::new(PackageLimits::default(), "1.0.0".parse().unwrap())
|
|
.inspect(
|
|
crate::support::package_with_contributions(crate::support::worker(), contributes),
|
|
None,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let digest = artifact.metadata.sha256.clone();
|
|
let instances = instances
|
|
.into_iter()
|
|
.map(|fixture| PluginInstance {
|
|
id: fixture.id.clone(),
|
|
name: fixture.id,
|
|
artifact_sha256: digest.clone(),
|
|
enabled: true,
|
|
trusted_process: true,
|
|
configuration: fixture.configuration,
|
|
secrets: BTreeMap::new(),
|
|
|
|
bindings: fixture.bindings,
|
|
revision: Revision::new(1).unwrap(),
|
|
})
|
|
.collect();
|
|
let store = Arc::new(Store {
|
|
artifacts: BTreeMap::from([(digest, artifact)]),
|
|
snapshot: std::sync::Mutex::new(PluginInstanceSnapshot {
|
|
config_revision: Revision::new(1).unwrap(),
|
|
instances,
|
|
}),
|
|
});
|
|
let runtime = PluginRuntime::new(
|
|
store.clone(),
|
|
store.clone(),
|
|
PluginRuntimeConfig {
|
|
cache_directory: cache.path().to_owned(),
|
|
host_version: "1.0.0".parse().unwrap(),
|
|
package_limits: PackageLimits::default(),
|
|
rpc_limits,
|
|
restart_circuit: Default::default(),
|
|
},
|
|
Arc::new(gateway_host::outbound::HttpClient::new().unwrap()),
|
|
Arc::new(gateway_host::process::ProcessSupervisor::new(
|
|
std::num::NonZeroUsize::new(16).unwrap(),
|
|
)),
|
|
);
|
|
(cache, store, runtime)
|
|
}
|
|
|
|
fn observation(id: &str, outcome: RequestObservationOutcome) -> RequestObservation {
|
|
observation_with_scope(
|
|
id,
|
|
outcome,
|
|
"client-key-observed",
|
|
vec![AccountGroupId::new("grp_11111111111111111111111111111111").unwrap()],
|
|
)
|
|
}
|
|
|
|
fn observation_with_scope(
|
|
id: &str,
|
|
outcome: RequestObservationOutcome,
|
|
client_key_id: &str,
|
|
account_group_ids: Vec<AccountGroupId>,
|
|
) -> RequestObservation {
|
|
let send_state = if outcome == RequestObservationOutcome::Rejected {
|
|
UpstreamSendState::NotSent
|
|
} else {
|
|
UpstreamSendState::Sent
|
|
};
|
|
RequestObservation::new(
|
|
ModelRequestId::new(id).unwrap(),
|
|
ConfigRevision::new(1).unwrap(),
|
|
RequestObservationScope::new(
|
|
ClientApiKeyId::new(client_key_id).unwrap(),
|
|
account_group_ids,
|
|
),
|
|
OperationKind::Generate,
|
|
outcome,
|
|
send_state,
|
|
std::time::SystemTime::now(),
|
|
)
|
|
.with_requested_model(PublicModelId::new("gpt-observed").unwrap())
|
|
.with_usage(Usage {
|
|
input_tokens: Some(11),
|
|
output_tokens: Some(7),
|
|
total_tokens: Some(18),
|
|
..Usage::default()
|
|
})
|
|
}
|
|
|
|
fn websocket_observation(
|
|
id: &str,
|
|
sequence: u64,
|
|
body: serde_json::Value,
|
|
) -> WebSocketResponseObservation {
|
|
WebSocketResponseObservation::new(
|
|
ModelRequestId::new(id).unwrap(),
|
|
ConfigRevision::new(1).unwrap(),
|
|
RequestObservationScope::new(
|
|
ClientApiKeyId::new("client-key-observed").unwrap(),
|
|
vec![AccountGroupId::new("grp_11111111111111111111111111111111").unwrap()],
|
|
),
|
|
OperationKind::Generate,
|
|
WebSocketResponseAttempt::new(
|
|
ProviderKind::new("openai").unwrap(),
|
|
ProviderAccountId::new("acct_observed").unwrap(),
|
|
NonZeroU32::new(2).unwrap(),
|
|
),
|
|
sequence,
|
|
ProtocolWireEvent::json("openai", Some("response.output_text.delta".into()), body).unwrap(),
|
|
)
|
|
.with_requested_model(PublicModelId::new("gpt-observed").unwrap())
|
|
}
|
|
|
|
async fn wait_for_lines(path: &std::path::Path, count: usize) -> Vec<serde_json::Value> {
|
|
tokio::time::timeout(Duration::from_secs(3), async {
|
|
loop {
|
|
let lines = std::fs::read_to_string(path)
|
|
.unwrap_or_default()
|
|
// 子进程先追加 JSON 再写换行;只解析已经提交完整行的记录
|
|
.split_inclusive('\n')
|
|
.filter(|line| line.ends_with('\n'))
|
|
.map(|line| serde_json::from_str(line).unwrap())
|
|
.collect::<Vec<_>>();
|
|
if lines.len() >= count {
|
|
return lines;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("observer marker")
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn frozen_plan_applies_scope_order_and_instance_deduplication() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let marker = marker_directory.path().join("observations.jsonl");
|
|
let context_marker = marker_directory.path().join("context.jsonl");
|
|
let (cache, _store, runtime) = setup(
|
|
vec![
|
|
(
|
|
"late",
|
|
serde_json::json!({"observation_label":"late","observation_marker":marker,"request_context_marker":context_marker}),
|
|
vec![binding(EventKind::RequestCompleted, 20)],
|
|
),
|
|
(
|
|
"early",
|
|
serde_json::json!({"observation_label":"early","observation_marker":marker,"request_context_marker":context_marker}),
|
|
vec![
|
|
binding(EventKind::RequestCompleted, 10),
|
|
binding(EventKind::WebSocketResponse, 10),
|
|
],
|
|
),
|
|
],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_observed", RequestObservationOutcome::Succeeded)
|
|
.with_extension_scope(
|
|
gateway_core::engine::extensions::ExtensionCallScope::default().with_request(
|
|
Arc::new(gateway_core::engine::extensions::RequestContext {
|
|
method: "POST".into(),
|
|
uri: "/v1/responses?observer=1".into(),
|
|
http_version: "HTTP/1.1".into(),
|
|
headers: vec![gateway_core::engine::middleware::MiddlewareHeader::new(
|
|
"x-binary",
|
|
vec![128, 255].into(),
|
|
)],
|
|
peer_addr: Some("[2001:db8::1]:42000".parse().unwrap()),
|
|
client_ip: Some("203.0.113.7".parse().unwrap()),
|
|
}),
|
|
),
|
|
)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
let lines = wait_for_lines(&marker, 2).await;
|
|
assert_eq!(
|
|
lines.len(),
|
|
2,
|
|
"one completed event carries both terminal and usage facts"
|
|
);
|
|
let contexts = wait_for_lines(&context_marker, 2).await;
|
|
for captured in &contexts {
|
|
assert_eq!(captured["request"]["uri"], "/v1/responses?observer=1");
|
|
assert_eq!(captured["request"]["peer_addr"], "[2001:db8::1]:42000");
|
|
assert_eq!(captured["request"]["client_ip"], "203.0.113.7");
|
|
let request: gateway_plugin_sdk::RequestContext =
|
|
serde_json::from_value(captured["request"].clone()).unwrap();
|
|
assert_eq!(request.headers[0].name, "x-binary");
|
|
assert_eq!(request.headers[0].value, [128, 255]);
|
|
}
|
|
assert_eq!(lines[0]["label"], "early");
|
|
assert!(lines[0]["observation"]["terminal"].is_object());
|
|
assert_eq!(lines[0]["observation"]["usage"]["total_tokens"], 18);
|
|
assert_eq!(lines[1]["label"], "late");
|
|
assert!(lines[1]["observation"]["terminal"].is_object());
|
|
assert_eq!(lines[1]["observation"]["usage"]["total_tokens"], 18);
|
|
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_model_miss", RequestObservationOutcome::Succeeded)
|
|
.with_requested_model(PublicModelId::new("gpt-other").unwrap())
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_rejected", RequestObservationOutcome::Rejected),
|
|
);
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
assert_eq!(wait_for_lines(&marker, 2).await.len(), 2);
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_observer_preserves_order_generation_and_complete_payload() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let hidden = marker_directory.path().join("hidden.jsonl");
|
|
let readable = marker_directory.path().join("readable.jsonl");
|
|
let (cache, _store, runtime) = setup_instances(
|
|
vec![
|
|
InstanceFixture {
|
|
id: "hidden".into(),
|
|
configuration: serde_json::json!({
|
|
"observation_label":"hidden",
|
|
"websocket_observation_marker":hidden,
|
|
}),
|
|
|
|
bindings: vec![binding(EventKind::WebSocketResponse, 1)],
|
|
},
|
|
InstanceFixture {
|
|
id: "readable".into(),
|
|
configuration: serde_json::json!({
|
|
"observation_label":"readable",
|
|
"websocket_observation_marker":readable,
|
|
}),
|
|
|
|
bindings: vec![binding(EventKind::WebSocketResponse, 2)],
|
|
},
|
|
],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
for (sequence, secret) in [(1, "first"), (2, "second")] {
|
|
plan.dispatch_websocket_response(
|
|
generation.clone(),
|
|
websocket_observation(
|
|
"req_websocket_ordered",
|
|
sequence,
|
|
serde_json::json!({"type":"response.output_text.delta","secret":secret}),
|
|
),
|
|
);
|
|
}
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation(
|
|
"req_websocket_ordered",
|
|
RequestObservationOutcome::Succeeded,
|
|
),
|
|
);
|
|
drop(plan);
|
|
drop(generation);
|
|
|
|
let hidden = wait_for_lines(&hidden, 2).await;
|
|
let readable = wait_for_lines(&readable, 2).await;
|
|
assert_eq!(
|
|
hidden
|
|
.iter()
|
|
.map(|line| line["observation"]["sequence"].as_u64().unwrap())
|
|
.collect::<Vec<_>>(),
|
|
vec![1, 2]
|
|
);
|
|
assert!(hidden.iter().all(|line| {
|
|
line["payload_bytes"].as_u64().unwrap() > 0
|
|
&& line["observation"]["payload_included"] == true
|
|
&& line["observation"]["event_type"] == "response.output_text.delta"
|
|
&& line["observation"]["account_id"] == "acct_observed"
|
|
}));
|
|
assert_eq!(
|
|
readable
|
|
.iter()
|
|
.map(|line| line["observation"]["sequence"].as_u64().unwrap())
|
|
.collect::<Vec<_>>(),
|
|
vec![1, 2]
|
|
);
|
|
assert!(readable.iter().all(|line| {
|
|
line["payload_bytes"].as_u64().unwrap() > 0
|
|
&& line["observation"]["payload_included"] == true
|
|
&& line["observation"]["event_type"] == "response.output_text.delta"
|
|
&& line["observation"]["account_id"] == "acct_observed"
|
|
}));
|
|
assert_eq!(
|
|
readable
|
|
.iter()
|
|
.map(|line| {
|
|
serde_json::from_str::<serde_json::Value>(line["payload_text"].as_str().unwrap())
|
|
.unwrap()["secret"]
|
|
.as_str()
|
|
.unwrap()
|
|
.to_owned()
|
|
})
|
|
.collect::<Vec<_>>(),
|
|
vec!["first", "second"]
|
|
);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_observer_bounds_event_count_and_payload_bytes_without_blocking() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let count_started = marker_directory.path().join("count-started.jsonl");
|
|
let count_completed = marker_directory.path().join("count-completed.jsonl");
|
|
let limits = RpcLimits {
|
|
maximum_calls: 1,
|
|
..RpcLimits::default()
|
|
};
|
|
let (count_cache, _count_store, count_runtime) = setup_instances(
|
|
vec![InstanceFixture {
|
|
id: "count".into(),
|
|
configuration: serde_json::json!({
|
|
"websocket_observation_started_marker":count_started,
|
|
"websocket_observation_marker":count_completed,
|
|
"websocket_observation_delay_ms":150,
|
|
}),
|
|
|
|
bindings: vec![binding(EventKind::WebSocketResponse, 1)],
|
|
}],
|
|
limits,
|
|
)
|
|
.await;
|
|
let count_generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&count_runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let count_plan = count_runtime
|
|
.observer_registry()
|
|
.resolve(&count_generation)
|
|
.unwrap();
|
|
count_plan.dispatch_websocket_response(
|
|
count_generation.clone(),
|
|
websocket_observation(
|
|
"req_websocket_count",
|
|
1,
|
|
serde_json::json!({"type":"response.delta","value":1}),
|
|
),
|
|
);
|
|
wait_for_lines(&count_started, 1).await;
|
|
for sequence in [2, 3] {
|
|
count_plan.dispatch_websocket_response(
|
|
count_generation.clone(),
|
|
websocket_observation(
|
|
"req_websocket_count",
|
|
sequence,
|
|
serde_json::json!({"type":"response.delta","value":sequence}),
|
|
),
|
|
);
|
|
}
|
|
count_plan.dispatch(
|
|
count_generation.clone(),
|
|
observation("req_websocket_count", RequestObservationOutcome::Succeeded),
|
|
);
|
|
drop(count_plan);
|
|
drop(count_generation);
|
|
let completed = wait_for_lines(&count_completed, 2).await;
|
|
assert_eq!(completed.len(), 2);
|
|
assert_eq!(completed[0]["observation"]["sequence"], 1);
|
|
assert_eq!(completed[1]["observation"]["sequence"], 2);
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
assert_eq!(wait_for_lines(&count_completed, 2).await.len(), 2);
|
|
super::wait_until_empty(count_cache.path()).await;
|
|
|
|
let byte_started = marker_directory.path().join("byte-started.jsonl");
|
|
let fault_completed = marker_directory.path().join("fault-completed.jsonl");
|
|
let healthy_completed = marker_directory.path().join("healthy-completed.jsonl");
|
|
let (byte_cache, _byte_store, byte_runtime) = setup_instances(
|
|
vec![
|
|
InstanceFixture {
|
|
id: "fault".into(),
|
|
configuration: serde_json::json!({
|
|
"websocket_observation_started_marker":byte_started,
|
|
"websocket_observation_marker":fault_completed,
|
|
"websocket_observation_delay_ms":150,
|
|
"websocket_observation_fault":true,
|
|
}),
|
|
|
|
bindings: vec![binding(EventKind::WebSocketResponse, 1)],
|
|
},
|
|
InstanceFixture {
|
|
id: "healthy".into(),
|
|
configuration: serde_json::json!({
|
|
"websocket_observation_marker":healthy_completed,
|
|
}),
|
|
|
|
bindings: vec![binding(EventKind::WebSocketResponse, 2)],
|
|
},
|
|
],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let byte_generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&byte_runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let byte_plan = byte_runtime
|
|
.observer_registry()
|
|
.resolve(&byte_generation)
|
|
.unwrap();
|
|
let large_body = || {
|
|
serde_json::json!({
|
|
"type":"response.delta",
|
|
"content":"x".repeat(600_000),
|
|
})
|
|
};
|
|
byte_plan.dispatch_websocket_response(
|
|
byte_generation.clone(),
|
|
websocket_observation("req_websocket_bytes", 1, large_body()),
|
|
);
|
|
wait_for_lines(&byte_started, 1).await;
|
|
byte_plan.dispatch_websocket_response(
|
|
byte_generation.clone(),
|
|
websocket_observation("req_websocket_bytes", 2, large_body()),
|
|
);
|
|
byte_plan.dispatch(
|
|
byte_generation.clone(),
|
|
observation("req_websocket_bytes", RequestObservationOutcome::Succeeded),
|
|
);
|
|
drop(byte_plan);
|
|
drop(byte_generation);
|
|
assert_eq!(wait_for_lines(&fault_completed, 1).await.len(), 1);
|
|
let healthy = wait_for_lines(&healthy_completed, 1).await;
|
|
assert_eq!(healthy.len(), 1, "观察失败不能阻断同批后续订阅者");
|
|
assert_eq!(healthy[0]["observation"]["sequence"], 1);
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
assert_eq!(wait_for_lines(&healthy_completed, 1).await.len(), 1);
|
|
super::wait_until_empty(byte_cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unused_observer_response_does_not_stop_the_shared_session() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let completed = marker_directory.path().join("healthy.jsonl");
|
|
let (cache, _store, runtime) = setup(
|
|
vec![
|
|
(
|
|
"invalid",
|
|
serde_json::json!({"invalid_response_method":"observer.observe"}),
|
|
vec![binding(EventKind::RequestCompleted, 1)],
|
|
),
|
|
(
|
|
"healthy",
|
|
serde_json::json!({"observation_marker":completed}),
|
|
vec![binding(EventKind::RequestCompleted, 2)],
|
|
),
|
|
],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_invalid_observer", RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
wait_for_lines(&completed, 1).await;
|
|
assert!(generation.is_ready());
|
|
assert!(generation.can_serve());
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn observer_failure_continues_the_plan_and_timeout_is_bounded() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let started = marker_directory.path().join("started.jsonl");
|
|
let completed = marker_directory.path().join("completed.jsonl");
|
|
let (cache, _store, runtime) = setup(
|
|
vec![
|
|
(
|
|
"fault",
|
|
serde_json::json!({
|
|
"observation_label":"fault",
|
|
"observation_started_marker":started,
|
|
"observation_marker":completed,
|
|
"observation_fault":true,
|
|
}),
|
|
vec![binding(EventKind::RequestCompleted, 1)],
|
|
),
|
|
(
|
|
"timeout",
|
|
serde_json::json!({
|
|
"observation_label":"timeout",
|
|
"observation_started_marker":started,
|
|
"observation_marker":completed,
|
|
"observation_delay_ms":2500,
|
|
}),
|
|
vec![binding(EventKind::RequestCompleted, 2)],
|
|
),
|
|
],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_failure", RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
let started_lines = wait_for_lines(&started, 2).await;
|
|
assert_eq!(started_lines[0]["label"], "fault");
|
|
assert_eq!(started_lines[1]["label"], "timeout");
|
|
tokio::time::sleep(Duration::from_millis(2100)).await;
|
|
let completed_lines = wait_for_lines(&completed, 1).await;
|
|
assert_eq!(completed_lines.len(), 1);
|
|
assert_eq!(completed_lines[0]["label"], "fault");
|
|
assert!(generation.is_ready(), "观察错误与超时不能击穿插件会话");
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn invalid_observer_stage_policy_and_mixed_order_are_rejected_before_publish() {
|
|
let mut invalid_stage = binding(EventKind::RequestCompleted, 1);
|
|
invalid_stage.stage = "routing".into();
|
|
let mut invalid_policy = binding(EventKind::RequestCompleted, 1);
|
|
invalid_policy.failure_policy = PluginFailurePolicy::Reject;
|
|
let mut invalid_websocket_stage = binding(EventKind::WebSocketResponse, 1);
|
|
invalid_websocket_stage.stage = "stream".into();
|
|
let mut invalid_websocket_policy = binding(EventKind::WebSocketResponse, 1);
|
|
invalid_websocket_policy.failure_policy = PluginFailurePolicy::Delegate;
|
|
let mut missing_event = binding(EventKind::RequestCompleted, 1);
|
|
missing_event.event = None;
|
|
let mut unknown_event = binding(EventKind::RequestCompleted, 1);
|
|
unknown_event.event = Some("unknown".into());
|
|
for bindings in [
|
|
vec![missing_event],
|
|
vec![unknown_event],
|
|
vec![
|
|
binding(EventKind::RequestCompleted, 1),
|
|
binding(EventKind::RequestCompleted, 1),
|
|
],
|
|
vec![invalid_stage],
|
|
vec![invalid_policy],
|
|
vec![invalid_websocket_stage],
|
|
vec![invalid_websocket_policy],
|
|
vec![
|
|
binding(EventKind::RequestCompleted, 1),
|
|
binding(EventKind::RequestCompleted, 2),
|
|
],
|
|
vec![
|
|
binding(EventKind::RequestCompleted, 1),
|
|
binding(EventKind::WebSocketResponse, 2),
|
|
],
|
|
] {
|
|
let (_cache, store, runtime) = setup(
|
|
vec![("invalid", serde_json::json!({}), bindings)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let snapshot = store.snapshot.lock().unwrap().clone();
|
|
assert!(
|
|
gateway_admin::ports::plugins::PluginPreparation::prepare(&runtime, snapshot)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn client_scope_filters_observations_and_exposes_only_the_key_identifier() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let marker = marker_directory.path().join("scoped-observations.jsonl");
|
|
let matching_group = "grp_11111111111111111111111111111111";
|
|
let other_group = "grp_22222222222222222222222222222222";
|
|
let mut scoped = binding(EventKind::RequestCompleted, 1);
|
|
scoped.client_key_ids = vec!["client-key-observed".into()];
|
|
scoped.account_group_ids = vec![matching_group.into()];
|
|
scoped.provider_ids.clear();
|
|
let (cache, _store, runtime) = setup(
|
|
vec![(
|
|
"scoped",
|
|
serde_json::json!({"observation_label":"scoped","observation_marker":marker}),
|
|
vec![scoped],
|
|
)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation_with_scope(
|
|
"req_scope_match",
|
|
RequestObservationOutcome::Rejected,
|
|
"client-key-observed",
|
|
vec![AccountGroupId::new(matching_group).unwrap()],
|
|
),
|
|
);
|
|
let lines = wait_for_lines(&marker, 1).await;
|
|
assert_eq!(lines[0]["observation"]["request_id"], "req_scope_match");
|
|
assert_eq!(lines[0]["observation"]["terminal"]["outcome"], "rejected");
|
|
assert_eq!(
|
|
lines[0]["observation"]["usage"]["failure"]["outcome"],
|
|
"rejected"
|
|
);
|
|
assert_eq!(
|
|
lines[0]["observation"]["usage"]["failure"]["send_state"],
|
|
"not_sent"
|
|
);
|
|
assert_eq!(
|
|
lines[0]["observation"]["usage"]["cost"]["status"],
|
|
"unknown"
|
|
);
|
|
let wire = lines[0]["observation"].to_string();
|
|
assert_eq!(
|
|
lines[0]["observation"]["client_key_id"],
|
|
"client-key-observed"
|
|
);
|
|
assert!(!wire.contains(matching_group));
|
|
|
|
for (request_id, key, groups) in [
|
|
(
|
|
"req_key_miss",
|
|
"client-key-other",
|
|
vec![AccountGroupId::new(matching_group).unwrap()],
|
|
),
|
|
(
|
|
"req_group_miss",
|
|
"client-key-observed",
|
|
vec![AccountGroupId::new(other_group).unwrap()],
|
|
),
|
|
("req_all_accounts_miss", "client-key-observed", vec![]),
|
|
] {
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation_with_scope(request_id, RequestObservationOutcome::Rejected, key, groups),
|
|
);
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
assert_eq!(wait_for_lines(&marker, 1).await.len(), 1);
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn completed_observation_contains_terminal_cost_timings_and_failure() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let marker = marker_directory.path().join("usage-observations.jsonl");
|
|
let (cache, _store, runtime) = setup(
|
|
vec![(
|
|
"usage",
|
|
serde_json::json!({"observation_label":"usage","observation_marker":marker}),
|
|
vec![binding(EventKind::RequestCompleted, 1)],
|
|
)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_usage_failure", RequestObservationOutcome::Failed)
|
|
.with_provider(ProviderKind::new("openai").unwrap())
|
|
.with_attempt_count(2)
|
|
.with_client_status_code(502)
|
|
.with_upstream_status_code(503)
|
|
.with_error_kind(GatewayErrorKind::UpstreamUnavailable)
|
|
.with_retry_after_ms(1_500)
|
|
.with_cost(
|
|
ProviderReportedCost::from_usd_ticks(123_000_000)
|
|
.unwrap()
|
|
.into_estimate(),
|
|
)
|
|
.with_timings(ModelRequestTimings {
|
|
first_text_ms: Some(12),
|
|
upstream_response_ms: Some(7_000),
|
|
upstream_engine_ms: Some(2.450638),
|
|
latency_ms: Some(34),
|
|
..ModelRequestTimings::default()
|
|
}),
|
|
);
|
|
let lines = wait_for_lines(&marker, 1).await;
|
|
let observed = &lines[0]["observation"];
|
|
assert_eq!(observed["terminal"]["outcome"], "failed");
|
|
assert_eq!(observed["usage"]["cost"]["status"], "known");
|
|
assert_eq!(observed["usage"]["cost"]["source"], "provider_reported");
|
|
assert_eq!(observed["usage"]["cost"]["total"]["amount"], "0.0123");
|
|
assert_eq!(observed["usage"]["cost"]["total"]["currency"], "USD");
|
|
assert_eq!(observed["usage"]["timings"]["first_text_ms"], 12);
|
|
assert_eq!(observed["usage"]["timings"]["upstream_response_ms"], 7_000);
|
|
assert_eq!(observed["usage"]["timings"]["upstream_engine_ms"], 2.450638);
|
|
assert_eq!(observed["usage"]["timings"]["latency_ms"], 34);
|
|
assert_eq!(observed["usage"]["failure"]["outcome"], "failed");
|
|
assert_eq!(observed["usage"]["failure"]["attempt_count"], 2);
|
|
assert_eq!(observed["usage"]["failure"]["client_status_code"], 502);
|
|
assert_eq!(observed["usage"]["failure"]["upstream_status_code"], 503);
|
|
assert_eq!(
|
|
observed["usage"]["failure"]["error_code"],
|
|
"upstream_unavailable"
|
|
);
|
|
assert_eq!(observed["usage"]["failure"]["retry_after_ms"], 1_500);
|
|
let wire = observed["usage"]["failure"].to_string();
|
|
assert!(!wire.contains("message"));
|
|
assert!(!wire.contains("raw_upstream_error"));
|
|
assert!(!wire.contains("provider_error_code"));
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn invalid_client_key_and_group_scopes_are_rejected_before_publish() {
|
|
let mut duplicate_keys = binding(EventKind::RequestCompleted, 1);
|
|
duplicate_keys.client_key_ids = vec!["client-key".into(), "client-key".into()];
|
|
let mut invalid_group = binding(EventKind::RequestCompleted, 1);
|
|
invalid_group.account_group_ids = vec!["not-a-group".into()];
|
|
let mut duplicate_groups = binding(EventKind::RequestCompleted, 1);
|
|
duplicate_groups.account_group_ids = vec![
|
|
"grp_33333333333333333333333333333333".into(),
|
|
"grp_33333333333333333333333333333333".into(),
|
|
];
|
|
for bindings in [
|
|
vec![duplicate_keys],
|
|
vec![invalid_group],
|
|
vec![duplicate_groups],
|
|
] {
|
|
let (_cache, store, runtime) = setup(
|
|
vec![("invalid-scope", serde_json::json!({}), bindings)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let snapshot = store.snapshot.lock().unwrap().clone();
|
|
assert!(
|
|
gateway_admin::ports::plugins::PluginPreparation::prepare(&runtime, snapshot)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn observer_call_load_tracks_the_active_lease_until_completion() {
|
|
use gateway_admin::ports::plugins::PluginRuntimeDiagnostics;
|
|
|
|
let markers = tempfile::tempdir().unwrap();
|
|
let started = markers.path().join("started.jsonl");
|
|
let completed = markers.path().join("completed.jsonl");
|
|
let (cache, store, runtime) = setup(
|
|
vec![(
|
|
"tracked",
|
|
serde_json::json!({
|
|
"observation_started_marker": started,
|
|
"observation_marker": completed,
|
|
"observation_delay_ms": 200,
|
|
}),
|
|
vec![binding(EventKind::RequestCompleted, 0)],
|
|
)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let snapshot = store.snapshot.lock().unwrap().clone();
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_tracked", RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
wait_for_lines(&started, 1).await;
|
|
let read_load = || async {
|
|
runtime
|
|
.runtime_diagnostics(&snapshot, Some(1), Some(&generation))
|
|
.await
|
|
.unwrap()
|
|
.remove("tracked")
|
|
.unwrap()
|
|
.load
|
|
.expect("published session has a load snapshot")
|
|
};
|
|
let load = read_load().await;
|
|
assert_eq!(load.active_calls, 1);
|
|
assert_eq!(load.executing_calls, 1);
|
|
assert_eq!(load.queued_calls, 0);
|
|
assert_eq!(load.pending_callbacks, 0);
|
|
assert_eq!(load.active_streams, 0);
|
|
|
|
wait_for_lines(&completed, 1).await;
|
|
tokio::time::timeout(Duration::from_secs(1), async {
|
|
loop {
|
|
let load = read_load().await;
|
|
if load.active_calls == 0 && load.executing_calls == 0 {
|
|
break;
|
|
}
|
|
tokio::task::yield_now().await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("completed calls release their observed resource occupancy");
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn observer_backpressure_drops_excess_and_inflight_dispatch_keeps_generation_alive() {
|
|
let marker_directory = tempfile::tempdir().unwrap();
|
|
let started = marker_directory.path().join("started.jsonl");
|
|
let completed = marker_directory.path().join("completed.jsonl");
|
|
let limits = RpcLimits {
|
|
maximum_calls: 1,
|
|
..RpcLimits::default()
|
|
};
|
|
let (cache, _store, runtime) = setup(
|
|
vec![(
|
|
"slow",
|
|
serde_json::json!({
|
|
"observation_label":"slow",
|
|
"observation_started_marker":started,
|
|
"observation_marker":completed,
|
|
"observation_delay_ms":200,
|
|
}),
|
|
vec![binding(EventKind::RequestCompleted, 1)],
|
|
)],
|
|
limits,
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_first", RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
wait_for_lines(&started, 1).await;
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation("req_excess", RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap()),
|
|
);
|
|
drop(plan);
|
|
drop(generation);
|
|
|
|
let completed = wait_for_lines(&completed, 1).await;
|
|
assert_eq!(completed.len(), 1);
|
|
assert_eq!(
|
|
completed[0]["observation"]["request_id"], "req_first",
|
|
"try-acquire backpressure must not queue a second observation"
|
|
);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn one_observer_matches_completed_and_websocket_scopes_independently() {
|
|
let markers = tempfile::tempdir().unwrap();
|
|
let completed = markers.path().join("completed.jsonl");
|
|
let websocket = markers.path().join("websocket.jsonl");
|
|
let mut completed_binding = binding(EventKind::RequestCompleted, 0);
|
|
completed_binding.models = vec!["gpt-completed".into()];
|
|
let mut websocket_binding = binding(EventKind::WebSocketResponse, 0);
|
|
websocket_binding.models = vec!["gpt-stream".into()];
|
|
let (cache, _store, runtime) = setup(
|
|
vec![(
|
|
"observer",
|
|
serde_json::json!({
|
|
"observation_marker": completed, "websocket_observation_marker": websocket,
|
|
}),
|
|
vec![completed_binding, websocket_binding],
|
|
)],
|
|
RpcLimits::default(),
|
|
)
|
|
.await;
|
|
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
|
&runtime,
|
|
ConfigRevision::new(1).unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
|
for (id, model) in [
|
|
("req_matched", "gpt-completed"),
|
|
("req_missed", "gpt-stream"),
|
|
] {
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation(id, RequestObservationOutcome::Succeeded)
|
|
.with_provider(ProviderKind::new("openai").unwrap())
|
|
.with_requested_model(PublicModelId::new(model).unwrap()),
|
|
);
|
|
}
|
|
for (id, model) in [
|
|
("req_matched_ws", "gpt-stream"),
|
|
("req_missed_ws", "gpt-completed"),
|
|
] {
|
|
plan.dispatch_websocket_response(
|
|
generation.clone(),
|
|
websocket_observation(id, 1, serde_json::json!({"delta":"test"}))
|
|
.with_requested_model(PublicModelId::new(model).unwrap()),
|
|
);
|
|
}
|
|
assert_eq!(
|
|
wait_for_lines(&completed, 1).await[0]["observation"]["request_id"],
|
|
"req_matched"
|
|
);
|
|
assert_eq!(
|
|
wait_for_lines(&websocket, 1).await[0]["observation"]["request_id"],
|
|
"req_matched_ws"
|
|
);
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
assert_eq!(wait_for_lines(&completed, 1).await.len(), 1);
|
|
assert_eq!(wait_for_lines(&websocket, 1).await.len(), 1);
|
|
for id in ["req_matched_ws", "req_missed_ws"] {
|
|
plan.dispatch(
|
|
generation.clone(),
|
|
observation(id, RequestObservationOutcome::Succeeded),
|
|
);
|
|
}
|
|
drop(plan);
|
|
drop(generation);
|
|
super::wait_until_empty(cache.path()).await;
|
|
}
|