Files
codex-proxy-rs/backend/crates/gateway-plugin/runtime/tests/generation/policy/mod.rs
T
ZyyandCodex 4d4c1e48ce feat(plugins)!: expose account binding and selection roles to schedulers
Scheduler v4 reports session ownership separately from native selection preferences and pins. Only eligible candidates are annotated, and cache preferences do not imply a session binding.

BREAKING CHANGE: scheduler plugins must declare capability version 4 and accept the new candidate fields.

Co-authored-by: Codex <noreply@openai.com>
2026-11-17 16:48:06 +08:00

2069 lines
73 KiB
Rust

//! 验证插件重试、模型路由与账号调度策略的顺序及宿主约束
mod compatibility;
mod facts;
mod http;
mod lifecycle;
mod mounts;
mod outbound;
mod retry;
mod scheduling;
mod service;
mod websocket;
use std::{
collections::{BTreeMap, BTreeSet, VecDeque},
num::{NonZeroU32, NonZeroUsize},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, SystemTime},
};
use bytes::Bytes;
use futures::future::BoxFuture;
use gateway_admin::{
model::{
Revision,
plugins::instances::{
PluginCapabilityBinding, PluginFailurePolicy, PluginInstance, PluginInstanceSnapshot,
},
},
ports::plugins::PluginPackageInspector,
};
use gateway_core::{
account::FastMode,
account::{
AccountCandidate, AccountEligibilityPolicy, AccountRuntimeSignals, AccountSelectionContext,
AccountSelectionPolicy, AccountWeight, CredentialRevision, CredentialState,
ProviderAccount, ProviderAccountId, QuotaState, RotationStrategy,
},
engine::{
ModelRequestId,
execution::ClientTransport,
extensions::ExtensionCallScope,
middleware::{
MiddlewareAuthority, MiddlewareBody, MiddlewareContext, MiddlewareError,
MiddlewareFrame, MiddlewareFraming, MiddlewareHeader, MiddlewareMount, MiddlewareNext,
MiddlewareRequest, MiddlewareResponse, MiddlewareTarget,
},
nested::ExecutionEffects,
policy::{ModelRouteDecision, RequestPolicyContext},
},
identity::ProviderKind,
lifecycle::CancellationToken,
operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload},
policy::ClientApiKeyId,
routing::extensions::{ExtensionPreparationPort, ExtensionSetReference},
routing::{AccountGroupId, ConfigRevision, PublicModelId},
};
use gateway_plugin_runtime::{
PackageInspector, PackageLimits, PluginRuntime, PluginRuntimeConfig, RpcLimits,
};
use gateway_plugin_sdk::{Capability, Contributions, Stage};
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
use crate::support::{environment::Environment, store::Store};
struct InstanceFixture {
id: &'static str,
configuration: serde_json::Value,
bindings: Vec<PluginCapabilityBinding>,
}
const MODEL_ROUTER_CONTRIBUTION: &str = "test.example.modelRouter";
const SCHEDULER_CONTRIBUTION: &str = "test.example.scheduler";
const MIDDLEWARE_CONTRIBUTION: &str = "test.example.middleware";
fn binding(
contribution: &str,
stage: &str,
order: i32,
failure_policy: PluginFailurePolicy,
) -> PluginCapabilityBinding {
PluginCapabilityBinding {
contribution: contribution.into(),
stage: stage.into(),
order,
failure_policy,
client_key_ids: vec![],
account_group_ids: vec![],
provider_ids: vec![],
models: vec![],
event: None,
identity_bindings: vec![],
}
}
async fn setup(instances: Vec<InstanceFixture>) -> (tempfile::TempDir, PluginRuntime) {
let contributes = Contributions::from([
crate::support::contribution(Capability::RetryPolicy, vec![Stage::Retry], vec![], vec![]),
crate::support::contribution(
Capability::ModelRouter,
vec![Stage::Routing],
vec![],
vec![],
),
crate::support::contribution(
Capability::Scheduler,
vec![Stage::Scheduling],
vec![],
vec![],
),
crate::support::contribution(
Capability::Middleware,
vec![Stage::Request, Stage::Attempt],
vec!["openai".into()],
vec!["openai".into()],
),
]);
setup_package(
instances,
crate::support::package_with_contributions(crate::support::worker(), contributes),
)
.await
}
async fn setup_package(
instances: Vec<InstanceFixture>,
package: Arc<[u8]>,
) -> (tempfile::TempDir, PluginRuntime) {
setup_package_with_limits(instances, package, RpcLimits::default()).await
}
async fn setup_package_with_limits(
instances: Vec<InstanceFixture>,
package: Arc<[u8]>,
rpc_limits: RpcLimits,
) -> (tempfile::TempDir, PluginRuntime) {
let cache = tempfile::tempdir().unwrap();
let artifact = PackageInspector::new(PackageLimits::default(), "1.0.0".parse().unwrap())
.inspect(package, None)
.await
.unwrap();
let digest = artifact.metadata.sha256.clone();
let instances = instances
.into_iter()
.map(|fixture| PluginInstance {
id: fixture.id.into(),
name: fixture.id.into(),
artifact_sha256: digest.clone(),
enabled: true,
trusted_process: true,
configuration: fixture.configuration,
secrets: BTreeMap::new(),
bindings: fixture.bindings,
revision: Revision::new(1).unwrap(),
})
.collect();
let store = Arc::new(Store {
artifacts: BTreeMap::from([(digest, artifact)]),
snapshot: std::sync::Mutex::new(PluginInstanceSnapshot {
config_revision: Revision::new(1).unwrap(),
instances,
}),
});
let runtime = PluginRuntime::new(
store.clone(),
store,
PluginRuntimeConfig {
cache_directory: cache.path().to_owned(),
host_version: "1.0.0".parse().unwrap(),
package_limits: PackageLimits::default(),
rpc_limits,
restart_circuit: Default::default(),
},
Arc::new(gateway_host::outbound::HttpClient::new().unwrap()),
Arc::new(gateway_host::process::ProcessSupervisor::new(
NonZeroUsize::new(32).unwrap(),
)),
);
(cache, runtime)
}
async fn prepare(runtime: &PluginRuntime) -> ExtensionSetReference {
ExtensionPreparationPort::prepare(runtime, ConfigRevision::new(1).unwrap())
.await
.unwrap()
}
fn policy_context(
runtime: &PluginRuntime,
generation: ExtensionSetReference,
request_id: &str,
) -> RequestPolicyContext {
RequestPolicyContext::new(
runtime.policy_registry().resolve(&generation).unwrap(),
generation,
ModelRequestId::new(request_id).unwrap(),
ClientApiKeyId::new("key-policy").unwrap(),
Vec::new(),
)
}
fn operation() -> Operation {
let context = serde_json::json!({
"opaque_request_headers": [
["x-feature", "b24="],
["authorization", "c2VjcmV0"],
["session_id", "c2VjcmV0"],
["thread_id", "c2VjcmV0"],
["X_Extension!#$%&'*+-.^_`|~", "b24="],
["x-plugin opaque", "not-base64?"],
]
})
.as_object()
.unwrap()
.clone();
Operation::Generate(GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object(
"openai",
serde_json::json!({"model":"public-a","input":"synthetic"})
.as_object()
.unwrap()
.clone(),
)
.unwrap()
.with_context(context),
))
}
#[tokio::test]
async fn retry_callback_preserves_entry_snapshot_and_ancestor_scope() {
use crate::support::request_context::{Dispatcher, request};
use gateway_core::{
engine::policy::{RetryDecision, RetryFacts},
error::ProviderErrorKind,
upstream::UpstreamSendState,
};
let (_cache, runtime) = setup(vec![InstanceFixture {
id: "retry-source",
configuration: serde_json::json!({"dispatch_on_method":"policy.retry_decision", "retry_decision":{"decision":"stop"}}),
bindings: vec![binding("test.example.retryPolicy", "retry", 0, PluginFailurePolicy::Delegate)],
}]).await;
let dispatcher = Arc::new(Dispatcher::default());
let port: Arc<dyn gateway_core::engine::middleware::http::Dispatcher> = dispatcher.clone();
runtime.bind_http(&port).unwrap();
let request = request();
let extensions = ExtensionCallScope::default()
.with_request(request.clone())
.extending("ancestor".into())
.unwrap();
let context = policy_context(&runtime, prepare(&runtime).await, "req_retry_child")
.with_extension_scope(extensions);
assert_eq!(
context
.retry_decision(RetryFacts {
attempt_index: NonZeroU32::MIN,
provider: ProviderKind::new("openai").unwrap(),
model: Some("gpt-test".into()),
error_kind: ProviderErrorKind::RateLimited,
upstream_status: Some(429),
send_state: UpstreamSendState::NotSent,
remaining_routing_attempts: 1,
remaining_deadline: Duration::from_secs(3),
retry_allowed: true,
})
.await,
RetryDecision::Stop
);
{
let calls = dispatcher.0.lock().unwrap();
assert_eq!(calls.len(), 1);
assert!(Arc::ptr_eq(
&calls[0].extensions.shared_request().unwrap(),
&request
));
assert!(calls[0].extensions.contains("ancestor"));
assert!(calls[0].extensions.contains("retry-source"));
assert_eq!(calls[0].request_id, "req_retry_child");
}
runtime.shutdown().await;
}
#[tokio::test]
async fn retry_policy_preserves_decisions_and_leaves_replay_safety_to_core() {
use gateway_core::{
engine::policy::{RetryDecision, RetryFacts},
error::ProviderErrorKind,
upstream::UpstreamSendState,
};
for (reply, allowed, expected) in [
(
serde_json::json!({"decision":"delegate"}),
true,
RetryDecision::Stop,
),
(
serde_json::json!({"decision":"retry"}),
true,
RetryDecision::Retry,
),
(
serde_json::json!({"decision":"retry"}),
false,
RetryDecision::Retry,
),
(
serde_json::json!({"decision":"retry","account_id":"acct_forbidden"}),
true,
RetryDecision::Retry,
),
] {
let records = tempfile::tempdir().unwrap();
let marker = records.path().join("retry.jsonl");
let (_cache, runtime) = setup(vec![
InstanceFixture {
id: "retry-first",
configuration: serde_json::json!({"retry_decision":reply,"retry_marker":marker}),
bindings: vec![binding(
"test.example.retryPolicy",
"retry",
0,
PluginFailurePolicy::Delegate,
)],
},
InstanceFixture {
id: "retry-second",
configuration: serde_json::json!({"retry_decision":{"decision":"stop"}}),
bindings: vec![binding(
"test.example.retryPolicy",
"retry",
1,
PluginFailurePolicy::Delegate,
)],
},
])
.await;
let context = policy_context(&runtime, prepare(&runtime).await, "req_retry");
let decision = context
.retry_decision(RetryFacts {
attempt_index: NonZeroU32::MIN,
provider: ProviderKind::new("openai").unwrap(),
model: Some("gpt-test".into()),
error_kind: ProviderErrorKind::RateLimited,
upstream_status: Some(429),
send_state: UpstreamSendState::NotSent,
remaining_routing_attempts: 1,
remaining_deadline: Duration::from_secs(3),
retry_allowed: allowed,
})
.await;
assert_eq!(decision, expected);
let facts = marker_lines(&marker, 1).await.remove(0);
assert_eq!(
facts["allowed_actions"],
if allowed {
serde_json::json!(["stop", "retry"])
} else {
serde_json::json!(["stop"])
}
);
assert_eq!(facts["error_kind"], "rate_limited");
assert!(facts.get("account_id").is_none());
runtime.shutdown().await;
}
}
#[tokio::test]
async fn retry_policy_combines_instance_budgets_with_one_request_bounded_chain() {
use gateway_core::{
engine::policy::{RetryDecision, RetryFacts},
error::ProviderErrorKind,
upstream::UpstreamSendState,
};
for (
first_budget,
first_delay,
second_budget,
second_delay,
request_budget,
expected,
reached_second,
) in [
(100, 500, 1000, 100, 2000, RetryDecision::Stop, true),
(1000, 100, 50, 300, 2000, RetryDecision::Delegate, true),
(1000, 300, 1000, 800, 2000, RetryDecision::Delegate, true),
(1000, 500, 1000, 0, 100, RetryDecision::Delegate, false),
(3000, 2100, 3000, 0, 5000, RetryDecision::Stop, true),
] {
let records = tempfile::tempdir().unwrap();
let second_marker = records.path().join("second.jsonl");
let (_cache, runtime) = setup(vec![
InstanceFixture {
id: "retry-delayed",
configuration: serde_json::json!({
"call_budgets_ms":{"retry":first_budget},
"retry_delay_ms":first_delay,
"retry_decision":{"decision":"delegate"},
}),
bindings: vec![binding(
"test.example.retryPolicy",
"retry",
0,
PluginFailurePolicy::Delegate,
)],
},
InstanceFixture {
id: "retry-next",
configuration: serde_json::json!({
"call_budgets_ms":{"retry":second_budget},
"retry_delay_ms":second_delay,
"retry_marker":second_marker,
"retry_decision":{"decision":"stop"},
}),
bindings: vec![binding(
"test.example.retryPolicy",
"retry",
1,
PluginFailurePolicy::Delegate,
)],
},
])
.await;
let context = policy_context(&runtime, prepare(&runtime).await, "req_retry_timeout");
let decision = tokio::time::timeout(
Duration::from_secs(6),
context.retry_decision(RetryFacts {
attempt_index: NonZeroU32::MIN,
provider: ProviderKind::new("openai").unwrap(),
model: Some("gpt-test".into()),
error_kind: ProviderErrorKind::RateLimited,
upstream_status: Some(429),
send_state: UpstreamSendState::NotSent,
remaining_routing_attempts: 1,
remaining_deadline: Duration::from_millis(request_budget),
retry_allowed: true,
}),
)
.await
.expect("重试链不能重置总预算");
assert_eq!(
decision, expected,
"{first_budget}/{second_budget}/{request_budget}"
);
assert_eq!(second_marker.exists(), reached_second);
runtime.shutdown().await;
}
}
async fn marker_lines(path: &std::path::Path, count: usize) -> Vec<serde_json::Value> {
tokio::time::timeout(Duration::from_secs(3), async {
loop {
let lines = std::fs::read_to_string(path)
.unwrap_or_default()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect::<Vec<_>>();
if lines.len() >= count {
return lines;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("plugin marker")
}
#[tokio::test]
async fn model_router_preserves_order_and_exposes_complete_request_data() {
let markers = tempfile::tempdir().unwrap();
let marker = markers.path().join("routes.jsonl");
let (cache, runtime) = setup(vec![
InstanceFixture {
id: "unhandled",
configuration: serde_json::json!({
"route_marker":marker,
"route_decision":{"decision":"unhandled"},
}),
bindings: vec![binding(
MODEL_ROUTER_CONTRIBUTION,
"routing",
1,
PluginFailurePolicy::Reject,
)],
},
InstanceFixture {
id: "handled",
configuration: serde_json::json!({
"route_marker":marker,
"route_decision":{"decision":"route","provider":"openai","model":"public-b"},
}),
bindings: vec![binding(
MODEL_ROUTER_CONTRIBUTION,
"routing",
2,
PluginFailurePolicy::Reject,
)],
},
])
.await;
let generation = prepare(&runtime).await;
let context = policy_context(&runtime, generation.clone(), "req_route");
let decision = context
.route_model(
operation(),
PublicModelId::new("public-a").unwrap(),
BTreeSet::from([
ProviderKind::new("openai").unwrap(),
ProviderKind::new("example").unwrap(),
]),
)
.await
.unwrap();
assert_eq!(
decision,
ModelRouteDecision::Route {
provider: Some(ProviderKind::new("openai").unwrap()),
model: Some(PublicModelId::new("public-b").unwrap()),
}
);
let lines = marker_lines(&marker, 2).await;
assert_eq!(lines[0]["body"], lines[1]["body"]);
assert_eq!(
lines[0]["request"]["headers"],
lines[1]["request"]["headers"]
);
assert_eq!(
serde_json::from_str::<serde_json::Value>(lines[1]["body"].as_str().unwrap()).unwrap(),
serde_json::json!({"model":"public-a","input":"synthetic"})
);
assert_eq!(
lines[1]["request"]["headers"],
serde_json::json!([
{"name":"x-feature","value_base64":"b24="},
{"name":"authorization","value_base64":"c2VjcmV0"},
{"name":"session_id","value_base64":"c2VjcmV0"},
{"name":"thread_id","value_base64":"c2VjcmV0"},
{"name":"x_extension!#$%&'*+-.^_`|~","value_base64":"b24="},
{"name":"x-plugin opaque","value_base64":"not-base64?"}
])
);
assert!(!lines[1].to_string().contains("secret"));
drop(context);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn model_router_can_delegate_after_a_managed_http_side_effect() {
let server = MockServer::start().await;
Mock::given(any())
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"ok":true})))
.expect(1)
.mount(&server)
.await;
let (cache, runtime) = setup(vec![
InstanceFixture {
id: "http-then-fault",
configuration: serde_json::json!({
"route_http_url":server.uri(),
"route_fault":true,
}),
bindings: vec![binding(
MODEL_ROUTER_CONTRIBUTION,
"routing",
1,
PluginFailurePolicy::Delegate,
)],
},
InstanceFixture {
id: "fallback",
configuration: serde_json::json!({
"route_decision":{"decision":"unhandled"},
}),
bindings: vec![binding(
MODEL_ROUTER_CONTRIBUTION,
"routing",
2,
PluginFailurePolicy::Reject,
)],
},
])
.await;
let generation = prepare(&runtime).await;
let context = policy_context(&runtime, generation.clone(), "req_route_http");
assert_eq!(
context
.route_model(
operation(),
PublicModelId::new("public-a").unwrap(),
BTreeSet::from([ProviderKind::new("openai").unwrap()]),
)
.await
.unwrap(),
ModelRouteDecision::Unhandled
);
drop(context);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
fn candidate(id: &str, weight: u16) -> AccountCandidate {
AccountCandidate {
inherited_concurrency: None,
account: ProviderAccount::new(
ProviderAccountId::new(id).unwrap(),
ProviderKind::new("example").unwrap(),
id.into(),
None,
"test".into(),
CredentialRevision::new(1).unwrap(),
Some(SystemTime::now() + Duration::from_secs(60)),
)
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
.with_scheduling(None, AccountWeight::new(weight).unwrap()),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_reset_windows: None,
quota_remaining_rank: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
}
}
fn selection_context() -> AccountSelectionContext {
AccountSelectionContext {
policy: AccountSelectionPolicy::new(
RotationStrategy::RoundRobin,
NonZeroU32::new(4).unwrap(),
),
now: SystemTime::now(),
excluded_accounts: BTreeSet::new(),
busy_accounts: Default::default(),
preferred_account: None,
preferred_account_overrides_weight: true,
round_robin_cursor: 0,
eligibility: AccountEligibilityPolicy::Enforce,
account_scope: None,
reserved_concurrency: 0,
}
}
#[tokio::test]
async fn scheduler_cannot_pick_outside_the_host_projected_priority_tier() {
let (cache, runtime) = setup(vec![InstanceFixture {
id: "scheduler",
configuration: serde_json::json!({"schedule_pick_index":0}),
bindings: vec![binding(
SCHEDULER_CONTRIBUTION,
"scheduling",
0,
PluginFailurePolicy::Reject,
)],
}])
.await;
let generation = prepare(&runtime).await;
let context = policy_context(&runtime, generation.clone(), "req_schedule");
let candidates = [candidate("acct_high", 100), candidate("acct_low", 10)];
let mut selection = selection_context();
let selected = context
.select_account(
NonZeroU32::MIN,
&ProviderKind::new("example").unwrap(),
Some("upstream-a"),
&candidates,
&mut selection,
Default::default(),
)
.await
.unwrap()
.unwrap();
assert_eq!(selected.candidate().account.id().as_str(), "acct_high");
drop(context);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
struct OneFrameBody {
frame: Option<MiddlewareFrame>,
reads: Arc<AtomicUsize>,
closes: Arc<AtomicUsize>,
}
impl MiddlewareBody for OneFrameBody {
fn next_frame(&mut self) -> BoxFuture<'_, Result<Option<MiddlewareFrame>, MiddlewareError>> {
Box::pin(async move {
self.reads.fetch_add(1, Ordering::Relaxed);
Ok(self.frame.take())
})
}
fn close(self: Box<Self>) -> BoxFuture<'static, ()> {
Box::pin(async move {
self.closes.fetch_add(1, Ordering::Relaxed);
})
}
}
struct Downstream {
calls: Arc<AtomicUsize>,
reads: Arc<AtomicUsize>,
closes: Arc<AtomicUsize>,
error: bool,
}
impl Downstream {
fn into_next(self) -> MiddlewareNext {
gateway_core::middleware::compose(Vec::new(), move |request| {
Box::pin(async move {
self.calls.fetch_add(1, Ordering::Relaxed);
assert_eq!(request.protocol(), "openai");
assert_eq!(request.body(), &Bytes::from_static(br#"{"input":"hello"}"#));
assert!(request.headers().iter().any(|header| {
header.name() == "authorization" && header.value().as_ref() == b"Bearer private"
}));
assert!(
request
.headers()
.iter()
.any(|header| header.name() == "x_team"
&& header.value().as_ref() == b"team")
);
let (status, framing, body) = if self.error {
(502, MiddlewareFraming::RawBytes, b"\0upstream".as_slice())
} else {
(
200,
MiddlewareFraming::JsonDocument,
br#"{"downstream":true}"#.as_slice(),
)
};
Ok(MiddlewareResponse::new(
"openai".into(),
status,
vec![
MiddlewareHeader::new(
"content-type",
Bytes::from_static(b"application/json"),
),
MiddlewareHeader::new("x-upstream-marker", Bytes::from_static(b"kept")),
],
Box::new(OneFrameBody {
reads: Arc::clone(&self.reads),
frame: Some(MiddlewareFrame::new(
Bytes::copy_from_slice(body),
framing,
true,
)),
closes: Arc::clone(&self.closes),
}),
))
})
})
}
}
fn middleware_context(transport: ClientTransport) -> MiddlewareContext {
middleware_context_with_deadline(
transport,
(SystemTime::now() + Duration::from_secs(5)).into(),
)
}
fn middleware_context_with_deadline(
transport: ClientTransport,
deadline: gateway_core::lifecycle::Deadline,
) -> MiddlewareContext {
MiddlewareContext::new(
MiddlewareTarget {
request_id: ModelRequestId::new("req_middleware").unwrap(),
mount: MiddlewareMount::Request,
attempt_index: None,
operation: Some(OperationKind::Generate),
endpoint: "/v1/responses".into(),
transport,
provider: None,
model: Some("public-a".into()),
account_id: None,
},
MiddlewareAuthority {
client_key_id: ClientApiKeyId::new("key-middleware").unwrap(),
account_group_ids: Arc::<[AccountGroupId]>::from([]),
cancellation: CancellationToken::new(),
deadline,
extension_scope: ExtensionCallScope::default(),
execution_effects: None,
},
)
}
fn attempt_middleware_context(effects: Arc<ExecutionEffects>) -> MiddlewareContext {
attempt_middleware_context_for_transport(effects, ClientTransport::HttpJson)
}
fn attempt_middleware_context_for_transport(
effects: Arc<ExecutionEffects>,
transport: ClientTransport,
) -> MiddlewareContext {
MiddlewareContext::new(
MiddlewareTarget {
request_id: ModelRequestId::new("req_attempt_http").unwrap(),
mount: MiddlewareMount::Attempt,
attempt_index: Some(NonZeroU32::MIN),
operation: Some(OperationKind::Generate),
endpoint: "/v1/responses".into(),
transport,
provider: Some(ProviderKind::new("openai").unwrap()),
model: Some("public-a".into()),
account_id: Some(ProviderAccountId::new("acct_openai").unwrap()),
},
MiddlewareAuthority {
client_key_id: ClientApiKeyId::new("key-middleware").unwrap(),
account_group_ids: Arc::<[AccountGroupId]>::from([]),
cancellation: CancellationToken::new(),
deadline: (SystemTime::now() + Duration::from_secs(5)).into(),
extension_scope: ExtensionCallScope::default(),
execution_effects: Some(effects),
},
)
}
fn middleware_headers() -> Vec<MiddlewareHeader> {
vec![
MiddlewareHeader::new("authorization", Bytes::from_static(b"Bearer private")),
MiddlewareHeader::new("X-OpenAI-Fedramp", Bytes::from_static(b"true")),
MiddlewareHeader::new(
"x-openai-account-routing-override",
Bytes::from_static(b"us_cr"),
),
MiddlewareHeader::new("connection", Bytes::from_static(b"x-hop")),
MiddlewareHeader::new("x-hop", Bytes::from_static(b"hidden")),
MiddlewareHeader::new("x_team", Bytes::from_static(b"team")),
]
}
#[tokio::test]
async fn attempt_middleware_can_delegate_after_a_managed_http_side_effect() {
let server = MockServer::start().await;
Mock::given(any())
.respond_with(ResponseTemplate::new(204))
.expect(1)
.mount(&server)
.await;
let (cache, runtime) = setup(vec![InstanceFixture {
id: "attempt-http",
configuration: serde_json::json!({
"middleware_http_url":server.uri(),
"middleware_fault_before_next":true,
}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"attempt",
0,
PluginFailurePolicy::Delegate,
)],
}])
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.expect("middleware plan");
let calls = Arc::new(AtomicUsize::new(0));
let closes = Arc::new(AtomicUsize::new(0));
let effects = Arc::new(ExecutionEffects::default());
let response = plan
.handle(
attempt_middleware_context(effects),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::default(),
error: false,
})
.into_next(),
)
.await
.unwrap();
assert_eq!(calls.load(Ordering::Relaxed), 1);
let (_, status, _, body, _) = response.into_parts();
assert_eq!(status, 200);
body.close().await;
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn middleware_next_and_lazy_body_mapping_use_one_real_rpc_call() {
let markers = tempfile::tempdir().unwrap();
let marker = markers.path().join("middleware.jsonl");
let (cache, runtime) = setup(vec![InstanceFixture {
id: "middleware",
configuration: serde_json::json!({
"middleware_marker":marker,
"middleware_map_body":true,
}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"request",
0,
PluginFailurePolicy::Reject,
)],
}])
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.expect("middleware plan");
let calls = Arc::new(AtomicUsize::new(0));
let closes = Arc::new(AtomicUsize::new(0));
let context = middleware_context(ClientTransport::HttpJson);
let response = plan
.handle(
context.clone(),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::default(),
error: false,
})
.into_next(),
)
.await
.unwrap();
assert_eq!(calls.load(Ordering::Relaxed), 1);
let (protocol, status, headers, mut body, envelope) = response.into_parts();
assert_eq!(protocol, "openai");
assert_eq!(status, 200);
assert!(headers.iter().any(|header| {
header.name() == "content-type" && header.value().as_ref() == b"application/json"
}));
assert!(envelope.is_none());
let frame = body.next_frame().await.unwrap().unwrap();
assert_eq!(frame.framing(), MiddlewareFraming::JsonDocument);
assert!(frame.terminal());
assert!(frame.transformed());
assert_eq!(
frame.into_bytes(),
Bytes::from_static(b"{\"downstream\":true} ")
);
assert!(body.next_frame().await.unwrap().is_none());
let first_marker = &marker_lines(&marker, 1).await[0];
assert_eq!(first_marker["body"], "{\"input\":\"hello\"}");
assert_eq!(
first_marker["headers"],
serde_json::Value::Array(
middleware_headers()
.iter()
.map(|header| serde_json::json!({
"name":header.name(), "value":header.value().to_vec(),
}))
.collect()
)
);
body.close().await;
tokio::time::timeout(Duration::from_secs(1), async {
while closes.load(Ordering::Relaxed) != 1 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let response = plan
.handle(
context,
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::default(),
error: false,
})
.into_next(),
)
.await
.unwrap();
let (_, _, _, body, _) = response.into_parts();
body.close().await;
tokio::time::timeout(Duration::from_secs(1), async {
while closes.load(Ordering::Relaxed) != 2 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(calls.load(Ordering::Relaxed), 2);
let response = plan
.handle(
middleware_context(ClientTransport::HttpSse),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::default(),
error: true,
})
.into_next(),
)
.await
.unwrap();
let (_, status, _, mut body, _) = response.into_parts();
assert_eq!(status, 502);
let frame = body.next_frame().await.unwrap().unwrap();
assert_eq!(frame.framing(), MiddlewareFraming::RawBytes);
assert!(frame.transformed());
assert_eq!(frame.into_bytes(), Bytes::from_static(b"\0upstream "));
body.close().await;
tokio::time::timeout(Duration::from_secs(1), async {
while closes.load(Ordering::Relaxed) != 3 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(calls.load(Ordering::Relaxed), 3);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn middleware_classifies_real_rpc_output_after_a_terminal_frame() {
for (id, extra_chunk, terminal_error) in [
("middleware-terminal-chunk", true, false),
("middleware-terminal-error", false, true),
] {
let (cache, runtime) = setup(vec![InstanceFixture {
id,
configuration: serde_json::json!({
"middleware_map_body": true,
"middleware_chunk_after_terminal": extra_chunk,
"middleware_error_after_terminal": terminal_error,
}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"request",
0,
PluginFailurePolicy::Reject,
)],
}])
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let calls = Arc::new(AtomicUsize::new(0));
let closes = Arc::new(AtomicUsize::new(0));
let response = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::default(),
error: false,
})
.into_next(),
)
.await
.unwrap();
let (_, _, _, mut body, _) = response.into_parts();
let error = body
.next_frame()
.await
.expect_err("output after the terminal frame must fail");
let expected = if terminal_error {
matches!(&error, MiddlewareError::Remote { .. })
} else {
matches!(&error, MiddlewareError::InvalidState)
};
assert!(
expected,
"unexpected terminal stream error for {id}: {error:?}"
);
body.close().await;
assert_eq!(calls.load(Ordering::Relaxed), 1);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
}
#[tokio::test]
async fn outer_plugin_receives_inner_rejection_details_and_preserves_rejection_status() {
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
let package = crate::support::package_with_contributions(
&worker,
Contributions::from([crate::support::contribution(
Capability::Middleware,
vec![Stage::Request],
vec!["openai".into()],
vec!["openai".into()],
)]),
);
let details = serde_json::json!({"reason":"fixture","disabled":false});
let (cache,runtime) = setup_package(vec![
InstanceFixture {id:"outer-error", configuration:serde_json::json!({"mode":"downstream_error","remote_error":true,"expected_message":"plugin rejection details","expected_details":details}), bindings:vec![binding(MIDDLEWARE_CONTRIBUTION,"request",0,PluginFailurePolicy::Reject)]},
InstanceFixture {id:"inner-error", configuration:serde_json::json!({"mode":"rejected"}), bindings:vec![binding(MIDDLEWARE_CONTRIBUTION,"request",1,PluginFailurePolicy::Reject)]},
],package).await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let result = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new("openai", middleware_headers(), Bytes::new()),
gateway_core::middleware::compose(Vec::new(), |_| {
Box::pin(async { panic!("rejection must not invoke terminal") })
}),
)
.await;
let Err(error) = result else {
panic!("expected rejection")
};
assert!(error.is_rejected());
assert!(generation.is_ready(), "主动拒绝请求不应停止插件");
let MiddlewareError::Remote { source, .. } = error else {
panic!("expected remote details")
};
let Some(gateway_plugin_runtime::RpcError::Remote(fault)) =
source.downcast_ref::<gateway_plugin_runtime::RpcError>()
else {
panic!("expected plugin error")
};
assert_eq!(fault.details.as_ref(), Some(&details));
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn real_request_plugin_receives_downstream_error_details_without_consuming_native_error() {
use gateway_core::{
error::{
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ProviderError,
ProviderErrorKind,
},
upstream::UpstreamSendState,
};
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
let package = crate::support::package_with_contributions(
&worker,
Contributions::from([crate::support::contribution(
Capability::Middleware,
vec![Stage::Request],
vec!["openai".into()],
vec!["openai".into()],
)]),
);
let message = "upstream fixture details: request exceeds quota";
let bytes = b"\0opaque upstream error\xff";
let (cache,runtime) = setup_package(vec![InstanceFixture {
id:"error-details",configuration:serde_json::json!({"mode":"downstream_error","expected_message":message,"expected_body":bytes.as_slice()}),
bindings:vec![binding(MIDDLEWARE_CONTRIBUTION,"request",0,PluginFailurePolicy::Reject)],
}],package).await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let result = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new("openai", middleware_headers(), Bytes::new()),
gateway_core::middleware::compose(Vec::new(), move |_| {
Box::pin(async move {
Err(MiddlewareError::Provider(
ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
.with_status(429)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
message,
Some("quota_fixture".into()),
Some("capacity".into()),
))
.with_client_visible_upstream_response(
ClientVisibleUpstreamResponse::new(
429,
Some(b"application/octet-stream".to_vec()),
Bytes::from_static(bytes),
),
),
))
})
}),
)
.await;
let Err(MiddlewareError::Provider(error)) = result else {
panic!("native Provider failure must survive the plugin boundary");
};
assert_eq!(error.send_state(), UpstreamSendState::Sent);
assert_eq!(
error
.client_visible_upstream_response()
.unwrap()
.body()
.as_ref(),
bytes
);
assert_eq!(
error.client_visible_upstream_error().unwrap().message(),
message
);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn real_request_plugins_rewrite_settings_in_onion_order() {
use gateway_core::{
policy::RateLimits, routing::request_settings::ExecutionSettings, settings::SettingsValues,
};
let profiles = |identity| {
BTreeMap::from([(
gateway_core::identity::ProviderKind::new("openai").unwrap(),
gateway_core::account::OpaqueProviderData::new(
serde_json::json!({"identity":identity})
.as_object()
.unwrap()
.clone(),
),
)])
};
let baseline = ExecutionSettings {
runtime: SettingsValues::new(3, "smart", BTreeMap::new(), None, None)
.with_request_profiles(profiles("key-default")),
fast_mode: FastMode::Disabled,
client_limits: RateLimits {
max_concurrency: 2,
requests_per_minute: 10,
},
timeout_ms: Some(60_000),
};
let baseline_json = serde_json::to_value(&baseline).unwrap();
let host_snapshot = gateway_core::routing::RuntimeSnapshot::new(
ConfigRevision::new(1).unwrap(),
gateway_core::settings::SettingsValues::new(1, "smart", Default::default(), None, None),
vec![],
vec![],
vec![],
)
.unwrap()
.with_settings(
&baseline
.runtime
.clone()
.with_request_profiles(profiles("host")),
)
.unwrap();
let configuration =
gateway_core::routing::request_settings::RequestSettings::new(Arc::new(host_snapshot))
.with_execution(
&gateway_core::policy::ClientPolicy::new(
middleware_context(ClientTransport::HttpJson)
.client_key_id()
.clone(),
gateway_core::policy::PlaintextClientApiKey::new("fixture-key").unwrap(),
Arc::new(
gateway_core::routing::FrozenAccountScope::new(
Arc::default(),
gateway_core::routing::ClientRoutingScope::all_accounts(),
)
.with_request_profiles(profiles("key-default"))
.with_fast_mode(baseline.fast_mode),
),
true,
baseline.client_limits,
),
baseline.timeout_ms,
);
let mut first = baseline_json.clone();
first["fast_mode"] = serde_json::json!("default");
first["runtime"]["max_account_rotations"] = serde_json::json!(0);
first["runtime"]["model_mappings"] = serde_json::json!({"alias":"model-one"});
let mut last = first.clone();
last["runtime"]["model_mappings"] = serde_json::json!({"alias":"model-two"});
last["timeout_ms"] = serde_json::json!(90_000);
// 显式写回宿主值也要记录覆盖,否则 Key 默认值会再次生效
last["runtime"]["request_profiles"] = serde_json::json!({"openai":{"identity":"host"}});
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
let package = crate::support::package_with_contributions(
&worker,
Contributions::from([crate::support::contribution(
Capability::Middleware,
vec![Stage::Request],
vec!["openai".into()],
vec!["openai".into()],
)]),
);
let (cache, runtime) = setup_package(vec![
InstanceFixture { id: "settings-first", configuration: serde_json::json!({"mode":"settings","expected_settings":baseline_json,"settings":first}), bindings: vec![binding(MIDDLEWARE_CONTRIBUTION,"request",0,PluginFailurePolicy::Reject)] },
InstanceFixture { id: "settings-second", configuration: serde_json::json!({"mode":"settings","expected_settings":first,"settings":last}), bindings: vec![binding(MIDDLEWARE_CONTRIBUTION,"request",1,PluginFailurePolicy::Reject)] },
InstanceFixture { id: "settings-pass", configuration: serde_json::json!({"mode":"settings","expected_settings":last}), bindings: vec![binding(MIDDLEWARE_CONTRIBUTION,"request",2,PluginFailurePolicy::Reject)] },
], package.clone()).await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
// 连续调用使用同一宿主基线,前一次插件改写不能泄漏到后一次
for _ in 0..2 {
let expected: ExecutionSettings = serde_json::from_value(last.clone()).unwrap();
let next = (Downstream {
calls: Arc::default(),
closes: Arc::default(),
reads: Arc::default(),
error: false,
})
.into_next();
let response = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
)
.with_settings(configuration.clone()),
gateway_core::middleware::compose(Vec::new(), move |request: MiddlewareRequest| {
Box::pin(async move {
let settings = request.settings().unwrap();
assert_eq!(settings.execution_values(), Some(expected));
let sources = settings.inspect();
assert_eq!(
sources["overrides"]["request_profiles"]["instance_id"],
"settings-second"
);
assert_eq!(
sources["overrides"]["model_mappings"]["instance_id"],
"settings-second"
);
assert_eq!(
sources["execution"]["fast_mode"]["instance_id"],
"settings-first"
);
assert_eq!(sources["execution"]["fast_mode"]["value"], "default");
assert_eq!(
sources["execution"]["timeout_ms"]["instance_id"],
"settings-second"
);
next.run(request).await
})
}),
)
.await
.unwrap();
let (_, _, _, mut body, _) = response.into_parts();
assert!(body.next_frame().await.unwrap().is_some());
assert!(body.next_frame().await.unwrap().is_none());
body.close().await;
}
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
// 根对象 null 也是显式替换,必须由设置合同拒绝,不能悄悄沿用原配置
let (cache, runtime) = setup_package(vec![InstanceFixture {
id: "settings-null",
configuration: serde_json::json!({"mode":"settings", "expected_settings":baseline_json, "settings":null}),
bindings: vec![binding(MIDDLEWARE_CONTRIBUTION, "request", 0, PluginFailurePolicy::Reject)],
}], package).await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let calls = Arc::new(AtomicUsize::new(0));
assert!(
plan.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#)
)
.with_settings(configuration),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::default(),
reads: Arc::default(),
error: false
})
.into_next(),
)
.await
.is_err()
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn sdk_middleware_entry_registers_and_maps_a_real_process_response() {
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
let package = crate::support::package_with_contributions(
&worker,
Contributions::from([crate::support::contribution(
Capability::Middleware,
vec![Stage::Request],
vec!["openai".into()],
vec!["openai".into()],
)]),
);
let (cache, runtime) = setup_package(
vec![InstanceFixture {
id: "sdk-middleware",
configuration: serde_json::json!({}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"request",
0,
PluginFailurePolicy::Reject,
)],
}],
package,
)
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let calls = Arc::new(AtomicUsize::new(0));
let closes = Arc::new(AtomicUsize::new(0));
let reads = Arc::new(AtomicUsize::new(0));
let response = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
closes: Arc::clone(&closes),
reads: Arc::clone(&reads),
error: false,
})
.into_next(),
)
.await
.unwrap();
let (_, status, headers, mut body, _) = response.into_parts();
assert_eq!(status, 200);
assert!(
headers.iter().any(|header| {
header.name() == "x-sdk-plugin" && header.value().as_ref() == b"active"
})
);
tokio::time::timeout(Duration::from_secs(5), async {
while reads.load(Ordering::Relaxed) == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
// 模拟慢消费者:SDK 可以继续发起读回调,但宿主不得预读第二个源
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(reads.load(Ordering::Relaxed), 1);
let frame = body.next_frame().await.unwrap().unwrap();
assert!(frame.terminal());
assert!(frame.transformed());
assert_eq!(
frame.into_bytes(),
Bytes::from_static(b"{\"downstream\":true} ")
);
assert!(body.next_frame().await.unwrap().is_none());
assert_eq!(reads.load(Ordering::Relaxed), 2);
body.close().await;
assert_eq!(calls.load(Ordering::Relaxed), 1);
tokio::time::timeout(Duration::from_secs(1), async {
while closes.load(Ordering::Relaxed) != 1 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
// 尚未消费源帧就关闭,等待源归还的回调必须一起取消并释放下游正文
reads.store(0, Ordering::Relaxed);
let response = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: Arc::clone(&calls),
reads: Arc::clone(&reads),
closes: Arc::clone(&closes),
error: false,
})
.into_next(),
)
.await
.unwrap();
let (_, _, _, body, _) = response.into_parts();
tokio::time::timeout(Duration::from_secs(5), async {
while reads.load(Ordering::Relaxed) == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(reads.load(Ordering::Relaxed), 1);
body.close().await;
tokio::time::timeout(Duration::from_secs(1), async {
while closes.load(Ordering::Relaxed) != 2 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(calls.load(Ordering::Relaxed), 2);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn core_execution_resolves_the_plugin_runtime_middleware_plan() {
let Some(environment) = Environment::create().await else {
return;
};
let marker = environment.directory.path().join("core-middleware.jsonl");
environment
.install_plugin(serde_json::json!({"middleware_marker": marker}))
.await;
let key_secret = "sk-core-middleware-fixture";
environment
.client_key("key_core_middleware", key_secret)
.await;
let (runtime, core) = environment.runtime().await;
let execution = core.execution_service();
let client = execution.authenticate(key_secret).unwrap();
let prepared = execution.prepare_execution(client).await.unwrap();
assert!(execution.middleware_plan(&prepared).is_some());
environment.release_plugin_accounts(&runtime);
drop(core);
drop(runtime);
environment.close().await;
}
#[tokio::test]
async fn instances_without_data_plane_bindings_publish_neither_plan() {
let (cache, runtime) = setup(vec![InstanceFixture {
id: "unbound",
configuration: serde_json::json!({}),
bindings: vec![],
}])
.await;
let generation = prepare(&runtime).await;
assert!(runtime.policy_registry().resolve(&generation).is_none());
assert!(
runtime
.execution_registry()
.middleware(&generation)
.is_none()
);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
struct TransparencyNext {
status: u16,
expected: MiddlewareRequest,
headers: Vec<MiddlewareHeader>,
frames: VecDeque<MiddlewareFrame>,
}
struct CapabilitiesNext(Arc<AtomicUsize>);
impl CapabilitiesNext {
fn into_next(self) -> MiddlewareNext {
gateway_core::middleware::compose(Vec::new(), move |request| {
Box::pin(async move {
self.0.fetch_add(1, Ordering::SeqCst);
let body = serde_json::from_slice(request.body()).unwrap();
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object("openai", body).unwrap(),
));
let operation = request.apply_capabilities(operation)?;
assert!(
!operation
.capability_requirements()
.features()
.contains(&gateway_core::operation::Feature::JsonSchema)
);
Ok(MiddlewareResponse::new(
"openai".into(),
200,
vec![],
Box::new(TransparencyBody(VecDeque::from([MiddlewareFrame::new(
Bytes::from_static(br#"{"output":[]}"#),
MiddlewareFraming::JsonDocument,
true,
)]))),
))
})
})
}
}
#[tokio::test]
async fn sdk_capability_declarations_require_request_stage() {
for (version, attempt) in [(6, false), (6, true)] {
let stage = if attempt {
Stage::Attempt
} else {
Stage::Request
};
let (capability, mut contribution) = crate::support::contribution(
Capability::Middleware,
vec![stage],
vec!["openai".into()],
vec!["openai".into()],
);
contribution.version = version;
let package = crate::support::package_with_contributions(
&std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap(),
Contributions::from([(capability, contribution)]),
);
let (_cache, runtime) = setup_package(vec![InstanceFixture {
id: "capability-declaration",
configuration: serde_json::json!({"mode":"declare","middleware_version":version,"attempt":attempt,
"body":{"model":"gpt-test","input":"return JSON matching the original schema"},
"capabilities":{"handled":["json_schema"],"required":[]}}),
bindings: vec![binding(MIDDLEWARE_CONTRIBUTION, if attempt { "attempt" } else { "request" }, 0, PluginFailurePolicy::Reject)],
}], package).await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let calls = Arc::new(AtomicUsize::new(0));
let context = if attempt {
attempt_middleware_context(Arc::default())
} else {
middleware_context(ClientTransport::HttpJson)
};
let result = plan.handle(context, MiddlewareRequest::new("openai", vec![], Bytes::from_static(
br#"{"model":"gpt-test","input":"answer","text":{"format":{"type":"json_schema","schema":{"type":"object"}}}}"#,
)), (CapabilitiesNext(calls.clone())).into_next()).await;
if !attempt {
let mut body = result.unwrap().into_parts().3;
assert!(body.next_frame().await.unwrap().unwrap().terminal());
assert!(body.next_frame().await.unwrap().is_none());
body.close().await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
} else {
assert!(result.is_err());
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
runtime.shutdown().await;
}
}
impl TransparencyNext {
fn into_next(self) -> MiddlewareNext {
gateway_core::middleware::compose(Vec::new(), move |request| {
Box::pin(async move {
assert_eq!(request.protocol(), self.expected.protocol());
assert_eq!(request.body(), self.expected.body());
assert_eq!(request.headers(), self.expected.headers());
Ok(MiddlewareResponse::new(
"openai".into(),
self.status,
self.headers,
Box::new(TransparencyBody(self.frames)),
))
})
})
}
}
struct TransparencyBody(VecDeque<MiddlewareFrame>);
impl MiddlewareBody for TransparencyBody {
fn next_frame(&mut self) -> BoxFuture<'_, Result<Option<MiddlewareFrame>, MiddlewareError>> {
Box::pin(async move { Ok(self.0.pop_front()) })
}
fn close(self: Box<Self>) -> BoxFuture<'static, ()> {
Box::pin(async {})
}
}
async fn assert_sdk_transparency(mode: &'static str) {
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
let package = crate::support::package_with_contributions(
&worker,
Contributions::from([crate::support::contribution(
Capability::Middleware,
vec![Stage::Request],
vec!["openai".into()],
vec!["openai".into()],
)]),
);
let (cache, runtime) = setup_package(
vec![InstanceFixture {
id: "transparency",
configuration: serde_json::json!({"mode": mode}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"request",
0,
PluginFailurePolicy::Reject,
)],
}],
package,
)
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let mut headers = middleware_headers();
headers.extend([
MiddlewareHeader::new("X-Repeated", Bytes::from_static(b"first")),
MiddlewareHeader::new("X-Repeated", Bytes::from_static(b"second")),
MiddlewareHeader::new("x-binary", Bytes::from_static(b"\x80\xff")),
]);
let response_headers = vec![
MiddlewareHeader::new("x-repeated", Bytes::from_static(b"first")),
MiddlewareHeader::new("x-repeated", Bytes::from_static(b"second")),
MiddlewareHeader::new("set-cookie", Bytes::from_static(b"test-hidden=kept")),
MiddlewareHeader::new("x-binary", Bytes::from_static(b"\x80\xff")),
];
let json = Bytes::from_static(
br#" { "z": 18446744073709551615, "unknown": { "x": 1e2 }, "a": "\u0061" } "#,
);
let cases = [
(ClientTransport::HttpJson, 200, MiddlewareFraming::JsonDocument, vec![json.clone()]),
(ClientTransport::WebSocket, 200, MiddlewareFraming::JsonDocument, vec![json.clone(), json.clone()]),
(ClientTransport::WebSocket, 503, MiddlewareFraming::JsonDocument, vec![json.clone()]),
(ClientTransport::InternalProbe, 200, MiddlewareFraming::JsonDocument, vec![json.clone()]),
(ClientTransport::InternalPlugin, 200, MiddlewareFraming::JsonDocument, vec![json.clone()]),
(ClientTransport::HttpJson, 503, MiddlewareFraming::RawBytes, vec![Bytes::from_static(b"unavailable")]),
(ClientTransport::HttpSse, 503, MiddlewareFraming::RawBytes, vec![Bytes::from_static(b"unavailable")]),
(ClientTransport::HttpSse, 200, MiddlewareFraming::SseEvent, vec![
Bytes::from_static(b": keep-alive\r\nevent: response.created\r\ndata: { \"z\": 1, \"a\": 2 }\r\n\r\n"),
Bytes::from_static(b"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{}}\n\n"),
]),
];
for (transport, status, framing, payloads) in cases {
for request_body in [json.clone(), Bytes::new()] {
let request = MiddlewareRequest::new("openai", headers.clone(), request_body.clone());
let mut expected_headers = headers.clone();
let mut expected_response_headers = response_headers.clone();
if mode == "headers" {
expected_headers.push(MiddlewareHeader::new(
"x-sdk-request",
Bytes::from_static(b"active"),
));
expected_response_headers.push(MiddlewareHeader::new(
"x-sdk-response",
Bytes::from_static(b"active"),
));
}
if mode == "credentials" {
expected_headers.retain(|header| header.name() != "authorization");
expected_headers.push(MiddlewareHeader::new(
"authorization",
Bytes::from_static(b"Bearer rewritten"),
));
expected_response_headers.retain(|header| header.name() != "set-cookie");
expected_response_headers.push(MiddlewareHeader::new(
"set-cookie",
Bytes::from_static(b"rewritten=true"),
));
}
let next = TransparencyNext {
status,
expected: MiddlewareRequest::new("openai", expected_headers, request_body),
headers: response_headers.clone(),
frames: payloads
.iter()
.enumerate()
.map(|(index, payload)| {
MiddlewareFrame::new(payload.clone(), framing, index + 1 == payloads.len())
})
.collect(),
};
let response = plan
.handle(middleware_context(transport), request, next.into_next())
.await
.unwrap();
let (protocol, actual_status, actual_headers, mut body, _) = response.into_parts();
assert_eq!(protocol, "openai");
assert_eq!(actual_status, status);
assert_eq!(actual_headers, expected_response_headers);
for (index, expected) in payloads.iter().enumerate() {
let frame = body.next_frame().await.unwrap().unwrap();
assert!(
!frame.transformed(),
"read-only inspection must retain provenance"
);
assert_eq!(frame.terminal(), index + 1 == payloads.len());
assert_eq!(frame.into_bytes(), *expected);
}
assert!(body.next_frame().await.unwrap().is_none());
body.close().await;
}
}
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
#[tokio::test]
async fn sdk_passthrough_preserves_original_bytes_and_all_headers() {
assert_sdk_transparency("passthrough").await;
assert_sdk_transparency("passthrough").await;
}
#[tokio::test]
async fn sdk_read_only_inspection_preserves_original_bytes_and_event_order() {
assert_sdk_transparency("inspect").await;
}
#[tokio::test]
async fn sdk_header_mutations_preserve_original_body_and_other_headers() {
assert_sdk_transparency("headers").await;
assert_sdk_transparency("credentials").await;
}
#[tokio::test]
async fn a_failed_plugin_rejects_only_its_bound_models_and_preserves_delegate_policy() {
for failure_policy in [PluginFailurePolicy::Reject, PluginFailurePolicy::Delegate] {
let mut mount = binding(
MODEL_ROUTER_CONTRIBUTION,
"routing",
0,
failure_policy.clone(),
);
mount.models = vec!["plugin-required".into()];
let (cache, runtime) = setup(vec![InstanceFixture {
id: "failed-router",
configuration: serde_json::json!({"startup":"fail"}),
bindings: vec![mount],
}])
.await;
let generation = prepare(&runtime).await;
let context = policy_context(&runtime, generation.clone(), "req_isolated_failure");
let unrelated = context
.route_model(
operation(),
PublicModelId::new("unrelated-model").unwrap(),
BTreeSet::new(),
)
.await;
assert_eq!(unrelated.unwrap(), ModelRouteDecision::Unhandled);
let required = context
.route_model(
operation(),
PublicModelId::new("plugin-required").unwrap(),
BTreeSet::new(),
)
.await;
if failure_policy == PluginFailurePolicy::Reject {
assert!(required.is_err());
} else {
assert_eq!(required.unwrap(), ModelRouteDecision::Unhandled);
}
drop(context);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
}
#[tokio::test]
async fn a_failed_middleware_preserves_scope_and_the_configured_failure_policy() {
for (model, failure_policy, expected_calls) in [
("unrelated", PluginFailurePolicy::Reject, 1),
("public-a", PluginFailurePolicy::Reject, 0),
("public-a", PluginFailurePolicy::Delegate, 1),
] {
let mut mount = binding(MIDDLEWARE_CONTRIBUTION, "request", 0, failure_policy);
mount.models = vec![model.into()];
let (cache, runtime) = setup(vec![InstanceFixture {
id: "failed-middleware",
configuration: serde_json::json!({"startup":"fail"}),
bindings: vec![mount],
}])
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
let calls = Arc::new(AtomicUsize::new(0));
let response = plan
.handle(
middleware_context(ClientTransport::HttpJson),
MiddlewareRequest::new(
"openai",
middleware_headers(),
Bytes::from_static(br#"{"input":"hello"}"#),
),
(Downstream {
calls: calls.clone(),
reads: Arc::default(),
closes: Arc::default(),
error: false,
})
.into_next(),
)
.await;
assert_eq!(response.is_ok(), expected_calls == 1);
assert_eq!(calls.load(Ordering::Relaxed), expected_calls);
drop(response);
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}
}
struct BodyCheckingNext {
calls: Arc<AtomicUsize>,
expected: Bytes,
}
impl BodyCheckingNext {
fn into_next(self) -> MiddlewareNext {
gateway_core::middleware::compose(Vec::new(), move |request| {
Box::pin(async move {
assert!(
request.body() == &self.expected,
"request body changed unexpectedly"
);
self.calls.fetch_add(1, Ordering::Relaxed);
Ok(MiddlewareResponse::new(
"openai".to_owned(),
200,
vec![],
Box::new(OneFrameBody {
reads: Arc::default(),
closes: Arc::default(),
frame: Some(MiddlewareFrame::new(
Bytes::from_static(b"{}"),
MiddlewareFraming::JsonDocument,
true,
)),
}),
))
})
})
}
}
#[tokio::test]
async fn middleware_preserves_large_body_and_following_requests() {
let (cache, runtime) = setup(vec![InstanceFixture {
id: "large-body",
configuration: serde_json::json!({}),
bindings: vec![binding(
MIDDLEWARE_CONTRIBUTION,
"request",
0,
PluginFailurePolicy::Reject,
)],
}])
.await;
let generation = prepare(&runtime).await;
let plan = runtime
.execution_registry()
.middleware(&generation)
.unwrap();
for length in [4 * 1024 * 1024 + 17, 32] {
let payload = Bytes::from(
(0..length)
.map(|index| (index % 251) as u8)
.collect::<Vec<_>>(),
);
let calls = Arc::new(AtomicUsize::new(0));
let response = plan
.handle(
middleware_context(ClientTransport::HttpSse),
MiddlewareRequest::new("openai", vec![], payload.clone()),
(BodyCheckingNext {
calls: Arc::clone(&calls),
expected: payload,
})
.into_next(),
)
.await
.unwrap();
let (_, status, _, mut body, _) = response.into_parts();
assert_eq!(status, 200);
assert_eq!(calls.load(Ordering::Relaxed), 1);
assert!(body.next_frame().await.unwrap().is_some());
assert!(body.next_frame().await.unwrap().is_none());
body.close().await;
}
drop(plan);
drop(generation);
super::wait_until_empty(cache.path()).await;
}