mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
refactor(backend): clarify ownership and align xai contracts
Separate execution, provider ports, administration, and plugin lifecycle responsibilities. Fix shutdown ordering, account lock reclamation, multipart cancellation, budget windows, and WebSocket close handling. Align xAI catalog, reasoning, tools, replay, quota, and cost handling with current Grok contracts and remove legacy fallbacks. Update API and architecture documentation with regression coverage. Validated backend suites, strict workspace Clippy, isolated PostgreSQL/Redis integration, and real Codex/Grok client flows. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -93,7 +93,7 @@ async fn execute(
|
||||
Some(plugin_runtime.clone()),
|
||||
Some(plugin_runtime.observer_registry()),
|
||||
Some(plugin_runtime.policy_registry()),
|
||||
Some(plugin_runtime.middleware_registry()),
|
||||
Some(plugin_runtime.execution_registry()),
|
||||
);
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
or_shutdown!(plugin_runtime, management.bind(&plugin_runtime));
|
||||
@@ -110,7 +110,7 @@ async fn execute(
|
||||
providers.admin,
|
||||
std::sync::Arc::new(gateway_host::pricing::ModelsDevPricing),
|
||||
);
|
||||
let mut services = gateway_admin::service::Registry::new(service_middleware(
|
||||
let mut services = gateway_admin::public_service::Registry::new(service_middleware(
|
||||
core.snapshots(),
|
||||
&plugin_runtime,
|
||||
));
|
||||
|
||||
@@ -72,11 +72,11 @@ impl ManagementPorts {
|
||||
pub(super) fn service_middleware(
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
runtime: &PluginRuntime,
|
||||
) -> gateway_admin::service::PlanSource {
|
||||
let middleware = runtime.middleware_registry();
|
||||
) -> gateway_admin::public_service::PlanSource {
|
||||
let middleware = runtime.execution_registry();
|
||||
Arc::new(move || {
|
||||
let snapshot = snapshots.snapshot_for_diagnostics()?;
|
||||
middleware.resolve(snapshot.extensions()?)
|
||||
middleware.middleware(snapshot.extensions()?)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
Some(plugin_runtime.clone()),
|
||||
Some(plugin_runtime.observer_registry()),
|
||||
Some(plugin_runtime.policy_registry()),
|
||||
Some(plugin_runtime.middleware_registry()),
|
||||
Some(plugin_runtime.execution_registry()),
|
||||
Some(plugin_runtime.frontend_authentication_registry()),
|
||||
);
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
@@ -122,14 +122,14 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
)
|
||||
.with_middleware({
|
||||
let snapshots = core.snapshots();
|
||||
let middleware = plugin_runtime.middleware_registry();
|
||||
let middleware = plugin_runtime.execution_registry();
|
||||
move |snapshot| {
|
||||
if let Some(snapshot) = snapshot {
|
||||
return middleware.resolve(snapshot.extensions()?);
|
||||
return middleware.middleware(snapshot.extensions()?);
|
||||
}
|
||||
// 数据面未就绪时,管理与诊断路由仍可使用发布候选中的插件
|
||||
let diagnostic = snapshots.snapshot_for_diagnostics()?;
|
||||
middleware.resolve(diagnostic.extensions()?)
|
||||
middleware.middleware(diagnostic.extensions()?)
|
||||
}
|
||||
});
|
||||
let http_dispatcher = api.dispatcher();
|
||||
|
||||
@@ -79,18 +79,25 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
path if path.starts_with("policy/") => {
|
||||
Some(&["account", "identity", "policy", "validation"])
|
||||
}
|
||||
"settings/values.rs" | "settings/compiled.rs" => {
|
||||
Some(&["account", "concurrency", "identity", "metering", "policy"])
|
||||
}
|
||||
path if path.starts_with("settings/") => Some(&[
|
||||
"account",
|
||||
"concurrency",
|
||||
"identity",
|
||||
"metering",
|
||||
"policy",
|
||||
"settings",
|
||||
]),
|
||||
path if path.starts_with("account/") => Some(&["account", "identity", "validation"]),
|
||||
_ => None,
|
||||
};
|
||||
let Some(allowed) = allowed else { continue };
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
let syntax = syn::parse_file(&source).expect("parse core source");
|
||||
let mut references = CrateReferences::default();
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
references.visit_file(&syntax);
|
||||
for dependency in references.0 {
|
||||
// 相对路径中的 self 指向当前 owner,不是跨 owner 依赖
|
||||
references.owners.remove(&references.module_path[0]);
|
||||
for dependency in references.owners {
|
||||
assert!(
|
||||
allowed.contains(&dependency.as_str()),
|
||||
"{} must not depend on crate::{dependency}",
|
||||
@@ -100,49 +107,155 @@ fn core_value_owners_do_not_depend_on_execution_or_routing() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_owner_paths_form_a_dag() {
|
||||
let root = backend_root().join("crates/gateway-core/src");
|
||||
let mut graph = BTreeMap::<String, BTreeSet<String>>::new();
|
||||
for relative in super::rust_files(&root) {
|
||||
if relative == Path::new("lib.rs") {
|
||||
continue;
|
||||
}
|
||||
let mut references = CrateReferences::in_module(&relative);
|
||||
let owner = references.module_path[0].clone();
|
||||
let source = fs::read_to_string(root.join(&relative)).expect("read core source");
|
||||
references.visit_file(&syn::parse_file(&source).expect("parse core source"));
|
||||
references.owners.remove(&owner);
|
||||
graph.entry(owner).or_default().extend(references.owners);
|
||||
}
|
||||
for owner in graph.keys() {
|
||||
let mut pending = graph[owner].iter().cloned().collect::<Vec<_>>();
|
||||
let mut visited = BTreeSet::new();
|
||||
while let Some(dependency) = pending.pop() {
|
||||
assert_ne!(
|
||||
&dependency, owner,
|
||||
"Core owner dependency cycle reaches {owner}"
|
||||
);
|
||||
if visited.insert(dependency.clone())
|
||||
&& let Some(next) = graph.get(&dependency)
|
||||
{
|
||||
pending.extend(next.iter().cloned());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_reference_scan_resolves_groups_reexports_and_relative_paths() {
|
||||
let syntax = syn::parse_file(
|
||||
r#"
|
||||
pub use crate::{engine::AttemptContext, settings::SettingsValues as Values};
|
||||
use super::super::runtime as publication;
|
||||
use self::compiled::Value;
|
||||
// crate::ignored::Comment
|
||||
const EXAMPLE: &str = "crate::ignored::Literal";
|
||||
"#,
|
||||
)
|
||||
.expect("parse reference fixture");
|
||||
let mut references = CrateReferences::in_module(Path::new("routing/snapshot.rs"));
|
||||
references.visit_file(&syntax);
|
||||
assert_eq!(
|
||||
references.owners,
|
||||
BTreeSet::from([
|
||||
"engine".to_owned(),
|
||||
"settings".to_owned(),
|
||||
"runtime".to_owned(),
|
||||
"routing".to_owned(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CrateReferences(BTreeSet<String>);
|
||||
struct CrateReferences {
|
||||
owners: BTreeSet<String>,
|
||||
module_path: Vec<String>,
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for CrateReferences {
|
||||
fn visit_path(&mut self, path: &'ast syn::Path) {
|
||||
let mut segments = path.segments.iter();
|
||||
if segments
|
||||
.next()
|
||||
.is_some_and(|segment| segment.ident == "crate")
|
||||
&& let Some(owner) = segments.next()
|
||||
{
|
||||
self.0.insert(owner.ident.to_string());
|
||||
}
|
||||
self.record(
|
||||
&path
|
||||
.segments
|
||||
.iter()
|
||||
.map(|part| part.ident.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
syn::visit::visit_path(self, path);
|
||||
}
|
||||
|
||||
fn visit_item_use(&mut self, item: &'ast syn::ItemUse) {
|
||||
self.use_tree(&item.tree, false);
|
||||
self.use_tree(&item.tree, &mut Vec::new());
|
||||
}
|
||||
}
|
||||
|
||||
impl CrateReferences {
|
||||
fn use_tree(&mut self, tree: &syn::UseTree, crate_root: bool) {
|
||||
fn in_module(relative: &Path) -> Self {
|
||||
let mut module = relative
|
||||
.with_extension("")
|
||||
.components()
|
||||
.map(|part| {
|
||||
part.as_os_str()
|
||||
.to_str()
|
||||
.expect("UTF-8 module path")
|
||||
.to_owned()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if module.last().is_some_and(|part| part == "mod") {
|
||||
module.pop();
|
||||
}
|
||||
Self {
|
||||
owners: BTreeSet::new(),
|
||||
module_path: module,
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&mut self, path: &[String]) {
|
||||
let Some(first) = path.first() else { return };
|
||||
let owner = if first == "crate" {
|
||||
path.get(1).cloned()
|
||||
} else if first == "self" || first == "super" {
|
||||
let mut module = self.module_path.clone();
|
||||
let mut rest = path;
|
||||
while rest
|
||||
.first()
|
||||
.is_some_and(|part| part == "super" || part == "self")
|
||||
{
|
||||
if rest[0] == "super" {
|
||||
module.pop();
|
||||
}
|
||||
rest = &rest[1..];
|
||||
}
|
||||
module.first().or_else(|| rest.first()).cloned()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(owner) = owner {
|
||||
self.owners.insert(owner);
|
||||
}
|
||||
}
|
||||
|
||||
fn use_tree(&mut self, tree: &syn::UseTree, prefix: &mut Vec<String>) {
|
||||
match tree {
|
||||
syn::UseTree::Path(path) if crate_root => {
|
||||
self.0.insert(path.ident.to_string());
|
||||
syn::UseTree::Path(path) => {
|
||||
prefix.push(path.ident.to_string());
|
||||
self.use_tree(&path.tree, prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Name(name) if crate_root => {
|
||||
self.0.insert(name.ident.to_string());
|
||||
syn::UseTree::Name(name) => {
|
||||
prefix.push(name.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Rename(rename) if crate_root => {
|
||||
self.0.insert(rename.ident.to_string());
|
||||
syn::UseTree::Rename(rename) => {
|
||||
prefix.push(rename.ident.to_string());
|
||||
self.record(prefix);
|
||||
prefix.pop();
|
||||
}
|
||||
syn::UseTree::Path(path) => self.use_tree(&path.tree, path.ident == "crate"),
|
||||
syn::UseTree::Group(group) => {
|
||||
for item in &group.items {
|
||||
self.use_tree(item, crate_root);
|
||||
self.use_tree(item, prefix);
|
||||
}
|
||||
}
|
||||
syn::UseTree::Glob(_) if crate_root => {
|
||||
self.0.insert("*".to_owned());
|
||||
}
|
||||
_ => {}
|
||||
syn::UseTree::Glob(_) => self.record(prefix),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -221,7 +334,7 @@ const ADAPTER_PUBLIC_MODULES: &[(&str, &[&str])] = &[
|
||||
"crates/providers/openai",
|
||||
&["config", "credential", "transport"],
|
||||
),
|
||||
("crates/providers/xai", &["credential", "transport"]),
|
||||
("crates/providers/xai", &["transport"]),
|
||||
];
|
||||
|
||||
/// 不对应单一生产模块、而是校验 crate/workspace 整体契约的根级测试场景
|
||||
|
||||
@@ -20,7 +20,7 @@ pub mod backup;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
pub mod service;
|
||||
pub mod public_service;
|
||||
mod use_case;
|
||||
pub use use_case::plugins::{PluginDistributionPorts, PluginManagementService, PluginsService};
|
||||
|
||||
@@ -205,7 +205,7 @@ pub enum AdminConfigError {
|
||||
#[derive(Clone)]
|
||||
pub struct AdminServices {
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
public_services: Arc<service::Registry>,
|
||||
public_services: Arc<public_service::Registry>,
|
||||
plugins: Arc<PluginsService>,
|
||||
plugin_management: Arc<PluginManagementService>,
|
||||
proxies: Arc<dyn ProxiesService>,
|
||||
@@ -231,7 +231,7 @@ impl AdminServices {
|
||||
self.timezone
|
||||
}
|
||||
|
||||
pub fn public_services(&self) -> Arc<service::Registry> {
|
||||
pub fn public_services(&self) -> Arc<public_service::Registry> {
|
||||
self.public_services.clone()
|
||||
}
|
||||
|
||||
@@ -331,7 +331,7 @@ impl AdminBundle {
|
||||
/// 组合根提供给控制面的运行能力;与配置和存储端口分别传入
|
||||
pub struct AdminRuntimePorts {
|
||||
pub timezone: gateway_core::time::DeploymentTimeZone,
|
||||
pub service_middleware: service::PlanSource,
|
||||
pub service_middleware: public_service::PlanSource,
|
||||
pub plugin_preparation: Arc<dyn ports::plugins::PluginPreparation>,
|
||||
pub plugin_management: Arc<dyn ports::plugin_management::PluginManagement>,
|
||||
pub published_snapshot: gateway_core::runtime::RuntimeSnapshotHandle,
|
||||
@@ -466,7 +466,7 @@ async fn initialize_inner(
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let mut public_services = service::Registry::new(service_middleware);
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
let services = AdminServices {
|
||||
timezone,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 插件管理页面、资源访问与授权回调的运行时端口
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::runtime::extensions::ExtensionSetReference;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
|
||||
use crate::model::{
|
||||
AdminError,
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::model::{
|
||||
PluginCompatibilityRequirements, PluginSource,
|
||||
},
|
||||
};
|
||||
use gateway_core::runtime::extensions::ExtensionSetReference;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
|
||||
/// Admin 准备完整候选并保活到提交后的发布结束;进程状态不能代替持久启用状态
|
||||
#[async_trait]
|
||||
|
||||
+3
-3
@@ -8,11 +8,11 @@ use std::{future::Future, sync::Arc};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::middleware::FrozenMiddlewarePlan,
|
||||
middleware::{
|
||||
compose,
|
||||
engine::middleware::{
|
||||
FrozenMiddlewarePlan,
|
||||
service::{Context, Error, Value},
|
||||
},
|
||||
middleware::compose,
|
||||
};
|
||||
|
||||
use crate::model::{AdminError, AdminErrorKind};
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
use crate::model::AdminError;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::middleware::service::{Error, Value};
|
||||
use gateway_core::engine::middleware::service::{Error, Value};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
//! 凭据变更共享的绑定校验、事务结算、失效与提交后观察
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext,
|
||||
accounts::DeleteAccounts,
|
||||
provider_credentials::{
|
||||
AuthorizationMutationTarget, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialDetails, CredentialMutationResult, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{provider::ProviderAdmin, store::AccountStore},
|
||||
};
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind, runtime::SnapshotControl};
|
||||
|
||||
use super::{map_store_error, publish_committed};
|
||||
|
||||
pub(super) async fn publish_credentials_and_observe_quota(
|
||||
provider: &std::sync::Arc<dyn crate::ports::provider::ProviderAdmin>,
|
||||
snapshot: &dyn SnapshotControl,
|
||||
revision: crate::model::Revision,
|
||||
account_ids: &[ProviderAccountId],
|
||||
request_id: &str,
|
||||
) -> Result<(), AdminError> {
|
||||
provider.account_facts_changed(account_ids).await;
|
||||
publish_committed(snapshot, revision).await?;
|
||||
|
||||
// 凭据已经提交;额度只是可重建的观察,不能拖住管理请求或回滚提交结果
|
||||
let provider = provider.clone();
|
||||
let account_ids = account_ids.to_vec();
|
||||
let request_id = request_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
provider = %provider.provider_kind().as_str(),
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn required_credential(
|
||||
accounts: &dyn AccountStore,
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDetails, AdminError> {
|
||||
accounts
|
||||
.credential_details(provider_kind, account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
}
|
||||
|
||||
pub(super) async fn required_plugin_credential(
|
||||
accounts: &dyn AccountStore,
|
||||
account_id: &ProviderAccountId,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDetails, AdminError> {
|
||||
accounts
|
||||
.credential_details_by_id(account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
}
|
||||
|
||||
pub(super) async fn import_proxy_binding(
|
||||
proxies: &dyn crate::ports::proxy::ProxyStore,
|
||||
id: Option<&str>,
|
||||
) -> Result<Option<crate::ports::proxy::ProxyImportReservation>, AdminError> {
|
||||
let Some(id) = id else { return Ok(None) };
|
||||
let reservation = proxies
|
||||
.reserve_import(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
Ok(Some(reservation))
|
||||
}
|
||||
|
||||
pub(super) async fn pending_authorization(
|
||||
accounts: &dyn AccountStore,
|
||||
proxies: &dyn crate::ports::proxy::ProxyStore,
|
||||
provider_kind: &ProviderKind,
|
||||
command: &StartAuthorization,
|
||||
resource: &'static str,
|
||||
) -> Result<PendingAuthorizationMutation, AdminError> {
|
||||
let proxy;
|
||||
let mut proxy_id = None;
|
||||
let target = match &command.reauthorization {
|
||||
Some(account_id) => {
|
||||
proxy = required_credential(accounts, provider_kind, account_id, resource)
|
||||
.await?
|
||||
.credential
|
||||
.outbound_proxy;
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
}
|
||||
None => {
|
||||
use crate::model::proxies::AccountProxySelection;
|
||||
proxy = match &command.outbound_proxy {
|
||||
Some(AccountProxySelection::Url(value)) => Some(value.clone()),
|
||||
Some(AccountProxySelection::Saved(id)) => {
|
||||
let record = proxies
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
proxy_id = Some(record.id);
|
||||
Some(record.proxy)
|
||||
}
|
||||
Some(AccountProxySelection::Direct) | None => None,
|
||||
};
|
||||
AuthorizationMutationTarget::Create {
|
||||
name: command.name.clone(),
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(PendingAuthorizationMutation::new(
|
||||
provider_kind.clone(),
|
||||
target,
|
||||
crate::model::provider_credentials::AuthorizationOwnerBinding::from_context(
|
||||
&command.context,
|
||||
),
|
||||
)
|
||||
.with_outbound_proxy(proxy)
|
||||
.with_outbound_proxy_id(proxy_id))
|
||||
}
|
||||
|
||||
pub(super) fn validate_prepared_import(
|
||||
provider_kind: &ProviderKind,
|
||||
prepared: &PreparedCredentialImport,
|
||||
_resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
if prepared.provider_kind != *provider_kind
|
||||
|| prepared
|
||||
.credentials
|
||||
.iter()
|
||||
.any(|credential| credential.provider_kind != *provider_kind)
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与请求范围不一致"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_prepared_rotation(
|
||||
account: &crate::model::accounts::AccountRecord,
|
||||
prepared: &PreparedCredentialRotation,
|
||||
_resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
validate_prepared_rotation_facts(account, prepared.facts())
|
||||
}
|
||||
|
||||
pub(super) fn validate_prepared_rotation_facts(
|
||||
account: &crate::model::accounts::AccountRecord,
|
||||
facts: &crate::model::provider_credentials::PreparedCredentialRotationFacts,
|
||||
) -> Result<(), AdminError> {
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与当前凭据不一致"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn validate_authorization_commit(
|
||||
provider_kind: &ProviderKind,
|
||||
context: &MutationContext,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
resource: &'static str,
|
||||
) -> Result<PreparedAuthorizationCommit, AdminError> {
|
||||
if prepared.pending.provider_kind() != provider_kind
|
||||
|| !prepared.pending.owner_binding().matches_context(context)
|
||||
{
|
||||
let validation_error = AdminError::conflict("OAuth 授权绑定已失效,请重新发起授权");
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
let matches_target = match (prepared.pending.target(), &prepared.credential) {
|
||||
(
|
||||
AuthorizationMutationTarget::Create { .. },
|
||||
PreparedAuthorizationCredential::Create(credential),
|
||||
) => credential.provider_kind == *provider_kind,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize { account_id },
|
||||
PreparedAuthorizationCredential::Reauthorize(credential),
|
||||
) => {
|
||||
let facts = credential.facts();
|
||||
facts.provider_kind == *provider_kind && facts.account_id == *account_id
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !matches_target {
|
||||
let validation_error = AdminError::conflict("OAuth 凭据与待提交目标不一致");
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(super) async fn commit_authorization(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
key: crate::model::provider_credentials::AuthorizationReceiptKey,
|
||||
settings: Option<crate::model::accounts::AccountImportSettings>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<crate::model::provider_credentials::AuthorizationCommitResult, AdminError> {
|
||||
if settings.is_some()
|
||||
&& matches!(
|
||||
&prepared.credential,
|
||||
PreparedAuthorizationCredential::Reauthorize(_)
|
||||
)
|
||||
{
|
||||
prepared.abort().await?;
|
||||
return Err(AdminError::invalid("重新授权不能修改账号设置"));
|
||||
}
|
||||
let crate::model::provider_credentials::AuthorizationCommitSettlement {
|
||||
command,
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
} = prepared.into_commit(settings, key);
|
||||
match accounts.commit_authorization(command, context).await {
|
||||
Ok(result) => {
|
||||
if let Some(guard) = credential_guard
|
||||
&& result.newly_committed
|
||||
{
|
||||
guard.finish();
|
||||
}
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(error) = guard.commit().await
|
||||
{
|
||||
// 账号与回执已经原子提交,Redis 清理失败不能把确定的成功改写为失败
|
||||
tracing::warn!(resource, settlement_error = %error, "authorization committed but pending cleanup failed");
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(settlement_error) = guard.abort().await
|
||||
{
|
||||
tracing::warn!(
|
||||
resource,
|
||||
store_error = %error,
|
||||
settlement_error = %settlement_error,
|
||||
"OAuth authorization claim release failed after Store commit failure"
|
||||
);
|
||||
return Err(settlement_error);
|
||||
}
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
settings: Option<crate::model::accounts::UpdateAccount>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
guard.finish();
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn commit_credential_refresh(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
guard.finish();
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn delete_credentials(
|
||||
accounts: &dyn AccountStore,
|
||||
provider: &dyn ProviderAdmin,
|
||||
command: CredentialDeletion,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDeletionResult, AdminError> {
|
||||
for account_id in &command.account_ids {
|
||||
required_credential(accounts, provider.provider_kind(), account_id, resource).await?;
|
||||
}
|
||||
let account_ids = command.account_ids;
|
||||
let revision = accounts
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: account_ids
|
||||
.iter()
|
||||
.map(|account_id| account_id.as_str().to_owned())
|
||||
.collect(),
|
||||
},
|
||||
&command.context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?;
|
||||
for account_id in &account_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
provider.account_facts_changed(&account_ids).await;
|
||||
Ok(CredentialDeletionResult {
|
||||
config_revision: revision,
|
||||
account_ids,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//! 管理端口错误到用例错误的统一分类与来源保留
|
||||
|
||||
use crate::model::{AdminError, AdminErrorKind};
|
||||
use crate::ports::store::{AdminStoreError, AdminStoreErrorKind};
|
||||
|
||||
pub(super) fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError {
|
||||
let kind = match error.kind() {
|
||||
AdminStoreErrorKind::Invalid => AdminErrorKind::Invalid,
|
||||
AdminStoreErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
AdminStoreErrorKind::StaleRevision
|
||||
| AdminStoreErrorKind::DuplicateName
|
||||
| AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "请求参数不合法",
|
||||
AdminErrorKind::NotFound => "请求的资源不存在",
|
||||
AdminErrorKind::Conflict => "当前资源状态冲突,请刷新后重试",
|
||||
AdminErrorKind::Unavailable => "依赖服务暂不可用",
|
||||
_ => "服务内部错误",
|
||||
};
|
||||
AdminError::new(kind, message).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn map_provider_error(
|
||||
error: crate::ports::provider::ProviderAdminError,
|
||||
resource: &'static str,
|
||||
) -> AdminError {
|
||||
use crate::ports::provider::ProviderAdminErrorKind;
|
||||
|
||||
let kind = match error.kind() {
|
||||
ProviderAdminErrorKind::Invalid => AdminErrorKind::Invalid,
|
||||
ProviderAdminErrorKind::Unsupported => AdminErrorKind::Invalid,
|
||||
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
ProviderAdminErrorKind::Ambiguous => AdminErrorKind::UpstreamResultUnknown,
|
||||
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::CredentialRefreshRequired => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "Provider 请求不合法",
|
||||
AdminErrorKind::NotFound => "Provider 资源不存在",
|
||||
AdminErrorKind::Conflict => "Provider 资源状态冲突,请刷新后重试",
|
||||
AdminErrorKind::BadGateway => "上游服务请求失败",
|
||||
AdminErrorKind::UpstreamResultUnknown => "上游执行结果未知,请刷新状态后再决定是否重试",
|
||||
AdminErrorKind::Unavailable => "Provider 服务暂不可用",
|
||||
AdminErrorKind::Internal => "服务内部错误",
|
||||
_ => "Provider 操作失败",
|
||||
};
|
||||
AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
// 用例的操作名补充端口资源上下文,底层分类与原始原因仍由来源持有
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("admin operation failed: {operation}")]
|
||||
struct OperationFailure {
|
||||
operation: &'static str,
|
||||
source: gateway_core::error::ErrorSource,
|
||||
}
|
||||
@@ -19,450 +19,16 @@ pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind, MutationContext,
|
||||
accounts::DeleteAccounts,
|
||||
provider_credentials::{
|
||||
AuthorizationMutationTarget, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialDetails, CredentialMutationResult, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdmin,
|
||||
store::{AccountStore, AdminStoreError, AdminStoreErrorKind},
|
||||
},
|
||||
mod credential_mutation;
|
||||
mod error;
|
||||
mod publication;
|
||||
|
||||
use credential_mutation::{
|
||||
commit_authorization, commit_credential_refresh, commit_credential_rotation,
|
||||
delete_credentials, import_proxy_binding, pending_authorization,
|
||||
publish_credentials_and_observe_quota, required_credential, required_plugin_credential,
|
||||
validate_authorization_commit, validate_prepared_import, validate_prepared_rotation,
|
||||
validate_prepared_rotation_facts,
|
||||
};
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
routing::{ConfigRevision, ProviderKind},
|
||||
runtime::SnapshotControl,
|
||||
};
|
||||
|
||||
fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError {
|
||||
let kind = match error.kind() {
|
||||
AdminStoreErrorKind::Invalid => AdminErrorKind::Invalid,
|
||||
AdminStoreErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
AdminStoreErrorKind::StaleRevision
|
||||
| AdminStoreErrorKind::DuplicateName
|
||||
| AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "请求参数不合法",
|
||||
AdminErrorKind::NotFound => "请求的资源不存在",
|
||||
AdminErrorKind::Conflict => "当前资源状态冲突,请刷新后重试",
|
||||
AdminErrorKind::Unavailable => "依赖服务暂不可用",
|
||||
_ => "服务内部错误",
|
||||
};
|
||||
AdminError::new(kind, message).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn map_provider_error(
|
||||
error: crate::ports::provider::ProviderAdminError,
|
||||
resource: &'static str,
|
||||
) -> AdminError {
|
||||
use crate::ports::provider::ProviderAdminErrorKind;
|
||||
|
||||
let kind = match error.kind() {
|
||||
ProviderAdminErrorKind::Invalid => AdminErrorKind::Invalid,
|
||||
ProviderAdminErrorKind::Unsupported => AdminErrorKind::Invalid,
|
||||
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
ProviderAdminErrorKind::Ambiguous => AdminErrorKind::UpstreamResultUnknown,
|
||||
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::CredentialRefreshRequired => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "Provider 请求不合法",
|
||||
AdminErrorKind::NotFound => "Provider 资源不存在",
|
||||
AdminErrorKind::Conflict => "Provider 资源状态冲突,请刷新后重试",
|
||||
AdminErrorKind::BadGateway => "上游服务请求失败",
|
||||
AdminErrorKind::UpstreamResultUnknown => "上游执行结果未知,请刷新状态后再决定是否重试",
|
||||
AdminErrorKind::Unavailable => "Provider 服务暂不可用",
|
||||
AdminErrorKind::Internal => "服务内部错误",
|
||||
_ => "Provider 操作失败",
|
||||
};
|
||||
AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn publish_committed(
|
||||
snapshot: &dyn SnapshotControl,
|
||||
revision: crate::model::Revision,
|
||||
) -> Result<(), AdminError> {
|
||||
let revision = ConfigRevision::new(revision.get())
|
||||
.map_err(|_| AdminError::internal("已提交的配置版本不合法"))?;
|
||||
snapshot.publish_committed(revision).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn publish_credentials_and_observe_quota(
|
||||
provider: &std::sync::Arc<dyn crate::ports::provider::ProviderAdmin>,
|
||||
snapshot: &dyn SnapshotControl,
|
||||
revision: crate::model::Revision,
|
||||
account_ids: &[ProviderAccountId],
|
||||
request_id: &str,
|
||||
) -> Result<(), AdminError> {
|
||||
provider.account_facts_changed(account_ids).await;
|
||||
publish_committed(snapshot, revision).await?;
|
||||
|
||||
// 凭据已经提交;额度只是可重建的观察,不能拖住管理请求或回滚提交结果
|
||||
let provider = provider.clone();
|
||||
let account_ids = account_ids.to_vec();
|
||||
let request_id = request_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
provider = %provider.provider_kind().as_str(),
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn required_credential(
|
||||
accounts: &dyn AccountStore,
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDetails, AdminError> {
|
||||
accounts
|
||||
.credential_details(provider_kind, account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
}
|
||||
|
||||
async fn required_plugin_credential(
|
||||
accounts: &dyn AccountStore,
|
||||
account_id: &ProviderAccountId,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDetails, AdminError> {
|
||||
accounts
|
||||
.credential_details_by_id(account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
}
|
||||
|
||||
async fn import_proxy_binding(
|
||||
proxies: &dyn crate::ports::proxy::ProxyStore,
|
||||
id: Option<&str>,
|
||||
) -> Result<Option<crate::ports::proxy::ProxyImportReservation>, AdminError> {
|
||||
let Some(id) = id else { return Ok(None) };
|
||||
let reservation = proxies
|
||||
.reserve_import(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
Ok(Some(reservation))
|
||||
}
|
||||
|
||||
async fn pending_authorization(
|
||||
accounts: &dyn AccountStore,
|
||||
proxies: &dyn crate::ports::proxy::ProxyStore,
|
||||
provider_kind: &ProviderKind,
|
||||
command: &StartAuthorization,
|
||||
resource: &'static str,
|
||||
) -> Result<PendingAuthorizationMutation, AdminError> {
|
||||
let proxy;
|
||||
let mut proxy_id = None;
|
||||
let target = match &command.reauthorization {
|
||||
Some(account_id) => {
|
||||
proxy = required_credential(accounts, provider_kind, account_id, resource)
|
||||
.await?
|
||||
.credential
|
||||
.outbound_proxy;
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
}
|
||||
None => {
|
||||
use crate::model::proxies::AccountProxySelection;
|
||||
proxy = match &command.outbound_proxy {
|
||||
Some(AccountProxySelection::Url(value)) => Some(value.clone()),
|
||||
Some(AccountProxySelection::Saved(id)) => {
|
||||
let record = proxies
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
proxy_id = Some(record.id);
|
||||
Some(record.proxy)
|
||||
}
|
||||
Some(AccountProxySelection::Direct) | None => None,
|
||||
};
|
||||
AuthorizationMutationTarget::Create {
|
||||
name: command.name.clone(),
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(PendingAuthorizationMutation::new(
|
||||
provider_kind.clone(),
|
||||
target,
|
||||
crate::model::provider_credentials::AuthorizationOwnerBinding::from_context(
|
||||
&command.context,
|
||||
),
|
||||
)
|
||||
.with_outbound_proxy(proxy)
|
||||
.with_outbound_proxy_id(proxy_id))
|
||||
}
|
||||
|
||||
fn validate_prepared_import(
|
||||
provider_kind: &ProviderKind,
|
||||
prepared: &PreparedCredentialImport,
|
||||
_resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
if prepared.provider_kind != *provider_kind
|
||||
|| prepared
|
||||
.credentials
|
||||
.iter()
|
||||
.any(|credential| credential.provider_kind != *provider_kind)
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与请求范围不一致"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_prepared_rotation(
|
||||
account: &crate::model::accounts::AccountRecord,
|
||||
prepared: &PreparedCredentialRotation,
|
||||
_resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
validate_prepared_rotation_facts(account, prepared.facts())
|
||||
}
|
||||
|
||||
fn validate_prepared_rotation_facts(
|
||||
account: &crate::model::accounts::AccountRecord,
|
||||
facts: &crate::model::provider_credentials::PreparedCredentialRotationFacts,
|
||||
) -> Result<(), AdminError> {
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与当前凭据不一致"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_authorization_commit(
|
||||
provider_kind: &ProviderKind,
|
||||
context: &MutationContext,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
resource: &'static str,
|
||||
) -> Result<PreparedAuthorizationCommit, AdminError> {
|
||||
if prepared.pending.provider_kind() != provider_kind
|
||||
|| !prepared.pending.owner_binding().matches_context(context)
|
||||
{
|
||||
let validation_error = AdminError::conflict("OAuth 授权绑定已失效,请重新发起授权");
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
let matches_target = match (prepared.pending.target(), &prepared.credential) {
|
||||
(
|
||||
AuthorizationMutationTarget::Create { .. },
|
||||
PreparedAuthorizationCredential::Create(credential),
|
||||
) => credential.provider_kind == *provider_kind,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize { account_id },
|
||||
PreparedAuthorizationCredential::Reauthorize(credential),
|
||||
) => {
|
||||
let facts = credential.facts();
|
||||
facts.provider_kind == *provider_kind && facts.account_id == *account_id
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !matches_target {
|
||||
let validation_error = AdminError::conflict("OAuth 凭据与待提交目标不一致");
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
async fn commit_authorization(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
key: crate::model::provider_credentials::AuthorizationReceiptKey,
|
||||
settings: Option<crate::model::accounts::AccountImportSettings>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<crate::model::provider_credentials::AuthorizationCommitResult, AdminError> {
|
||||
if settings.is_some()
|
||||
&& matches!(
|
||||
&prepared.credential,
|
||||
PreparedAuthorizationCredential::Reauthorize(_)
|
||||
)
|
||||
{
|
||||
prepared.abort().await?;
|
||||
return Err(AdminError::invalid("重新授权不能修改账号设置"));
|
||||
}
|
||||
let crate::model::provider_credentials::AuthorizationCommitSettlement {
|
||||
command,
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
} = prepared.into_commit(settings, key);
|
||||
match accounts.commit_authorization(command, context).await {
|
||||
Ok(result) => {
|
||||
if let Some(guard) = credential_guard
|
||||
&& result.newly_committed
|
||||
{
|
||||
guard.finish();
|
||||
}
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(error) = guard.commit().await
|
||||
{
|
||||
// 账号与回执已经原子提交,Redis 清理失败不能把确定的成功改写为失败
|
||||
tracing::warn!(resource, settlement_error = %error, "authorization committed but pending cleanup failed");
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(settlement_error) = guard.abort().await
|
||||
{
|
||||
tracing::warn!(
|
||||
resource,
|
||||
store_error = %error,
|
||||
settlement_error = %settlement_error,
|
||||
"OAuth authorization claim release failed after Store commit failure"
|
||||
);
|
||||
return Err(settlement_error);
|
||||
}
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_credential_rotation(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
settings: Option<crate::model::accounts::UpdateAccount>,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_rotation(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
guard.finish();
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_credential_refresh(
|
||||
accounts: &dyn AccountStore,
|
||||
prepared: PreparedCredentialRotation,
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (facts, guard) = prepared.into_parts();
|
||||
match accounts
|
||||
.commit_credential_refresh(
|
||||
CredentialRotationCommit {
|
||||
prepared: facts,
|
||||
settings: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
guard.finish();
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_credentials(
|
||||
accounts: &dyn AccountStore,
|
||||
provider: &dyn ProviderAdmin,
|
||||
command: CredentialDeletion,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialDeletionResult, AdminError> {
|
||||
for account_id in &command.account_ids {
|
||||
required_credential(accounts, provider.provider_kind(), account_id, resource).await?;
|
||||
}
|
||||
let account_ids = command.account_ids;
|
||||
let revision = accounts
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: account_ids
|
||||
.iter()
|
||||
.map(|account_id| account_id.as_str().to_owned())
|
||||
.collect(),
|
||||
},
|
||||
&command.context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?;
|
||||
for account_id in &account_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
provider.account_facts_changed(&account_ids).await;
|
||||
Ok(CredentialDeletionResult {
|
||||
config_revision: revision,
|
||||
account_ids,
|
||||
})
|
||||
}
|
||||
|
||||
// 用例的操作名补充端口资源上下文,底层分类与原始原因仍由来源持有
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("admin operation failed: {operation}")]
|
||||
struct OperationFailure {
|
||||
operation: &'static str,
|
||||
source: gateway_core::error::ErrorSource,
|
||||
}
|
||||
use error::{map_provider_error, map_store_error};
|
||||
use publication::publish_committed;
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_core::runtime::{RuntimeSnapshotHandle, extensions::ExtensionSetReference};
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
use gateway_core::runtime::RuntimeSnapshotHandle;
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
//! 管理事务提交后的运行快照发布入口
|
||||
|
||||
use crate::model::AdminError;
|
||||
use gateway_core::{routing::ConfigRevision, runtime::SnapshotControl};
|
||||
|
||||
pub(super) async fn publish_committed(
|
||||
snapshot: &dyn SnapshotControl,
|
||||
revision: crate::model::Revision,
|
||||
) -> Result<(), AdminError> {
|
||||
let revision = ConfigRevision::new(revision.get())
|
||||
.map_err(|_| AdminError::internal("已提交的配置版本不合法"))?;
|
||||
snapshot.publish_committed(revision).await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -107,7 +107,7 @@ pub(super) struct AdminHarness {
|
||||
plugin_store: Arc<dyn gateway_admin::ports::plugins::PluginStore>,
|
||||
plugin_inspector: Arc<dyn gateway_admin::ports::plugins::PluginPackageInspector>,
|
||||
client_key_verifier: Arc<dyn ClientKeyVerifier>,
|
||||
service_middleware: gateway_admin::service::PlanSource,
|
||||
service_middleware: gateway_admin::public_service::PlanSource,
|
||||
}
|
||||
|
||||
impl AdminHarness {
|
||||
@@ -194,7 +194,10 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn service_middleware(mut self, source: gateway_admin::service::PlanSource) -> Self {
|
||||
pub(super) fn service_middleware(
|
||||
mut self,
|
||||
source: gateway_admin::public_service::PlanSource,
|
||||
) -> Self {
|
||||
self.service_middleware = source;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -24,12 +24,10 @@ use gateway_admin::{
|
||||
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult},
|
||||
},
|
||||
};
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::{
|
||||
routing::ConfigRevision,
|
||||
runtime::{
|
||||
RuntimeSnapshotHandle, SnapshotControl,
|
||||
extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
},
|
||||
runtime::{RuntimeSnapshotHandle, SnapshotControl},
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -117,7 +115,7 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for Fixture {
|
||||
&self,
|
||||
_: &gateway_admin::model::plugins::instances::PluginInstanceSnapshot,
|
||||
_: Option<u64>,
|
||||
_: Option<&gateway_core::runtime::extensions::ExtensionSetReference>,
|
||||
_: Option<&gateway_core::routing::extensions::ExtensionSetReference>,
|
||||
) -> Option<
|
||||
std::collections::BTreeMap<
|
||||
String,
|
||||
@@ -132,7 +130,7 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for Fixture {
|
||||
impl gateway_admin::ports::plugins::PluginStateLifecycle for Fixture {
|
||||
async fn activate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::instances::PluginInstance,
|
||||
) -> Result<(), AdminError> {
|
||||
Ok(())
|
||||
@@ -142,7 +140,7 @@ impl gateway_admin::ports::plugins::PluginStateLifecycle for Fixture {
|
||||
}
|
||||
async fn migrate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: gateway_admin::model::plugins::state::PluginStateTransition,
|
||||
) -> Result<(), AdminError> {
|
||||
panic!("unexpected state migration")
|
||||
|
||||
@@ -40,12 +40,10 @@ use gateway_admin::{
|
||||
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult},
|
||||
},
|
||||
};
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::{
|
||||
routing::ConfigRevision,
|
||||
runtime::{
|
||||
RuntimeSnapshotHandle, SnapshotControl,
|
||||
extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
},
|
||||
runtime::{RuntimeSnapshotHandle, SnapshotControl},
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
|
||||
@@ -111,7 +111,7 @@ impl gateway_admin::ports::proxy::ProxyStore for TestPluginPorts {
|
||||
impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPorts {
|
||||
async fn validate_target(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
) -> Result<(), AdminError> {
|
||||
Err(AdminError::not_found("unused plugin fixture"))
|
||||
@@ -119,7 +119,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
|
||||
async fn start_callback(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: gateway_admin::model::plugins::management::StartPluginManagementCallback,
|
||||
_: &gateway_admin::model::auth::AdminRequestContext,
|
||||
@@ -129,7 +129,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn callback(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: &str,
|
||||
_: gateway_admin::model::plugins::management::PluginManagementRequest,
|
||||
@@ -139,14 +139,14 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn views(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
) -> Result<Vec<gateway_admin::model::plugins::management::PluginManagementView>, AdminError>
|
||||
{
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn resource(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: &str,
|
||||
_: bool,
|
||||
@@ -156,7 +156,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn handle(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: gateway_admin::model::plugins::management::PluginManagementRequest,
|
||||
) -> Result<gateway_admin::model::plugins::management::PluginManagementResponse, AdminError>
|
||||
@@ -171,7 +171,7 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for TestPluginPorts
|
||||
&self,
|
||||
_: &gateway_admin::model::plugins::instances::PluginInstanceSnapshot,
|
||||
_: Option<u64>,
|
||||
_: Option<&gateway_core::runtime::extensions::ExtensionSetReference>,
|
||||
_: Option<&gateway_core::routing::extensions::ExtensionSetReference>,
|
||||
) -> Option<
|
||||
std::collections::BTreeMap<
|
||||
String,
|
||||
@@ -186,7 +186,7 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for TestPluginPorts
|
||||
impl gateway_admin::ports::plugins::PluginStateLifecycle for TestPluginPorts {
|
||||
async fn activate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::instances::PluginInstance,
|
||||
) -> Result<(), AdminError> {
|
||||
panic!("unexpected state activation")
|
||||
@@ -196,7 +196,7 @@ impl gateway_admin::ports::plugins::PluginStateLifecycle for TestPluginPorts {
|
||||
}
|
||||
async fn migrate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: gateway_admin::model::plugins::state::PluginStateTransition,
|
||||
) -> Result<(), AdminError> {
|
||||
panic!("unexpected state migration")
|
||||
@@ -222,7 +222,7 @@ impl gateway_admin::ports::plugins::PluginPreparation for TestPluginPorts {
|
||||
async fn prepare(
|
||||
&self,
|
||||
_: gateway_admin::model::plugins::instances::PluginInstanceSnapshot,
|
||||
) -> Result<gateway_core::runtime::extensions::ExtensionSetReference, AdminError> {
|
||||
) -> Result<gateway_core::routing::extensions::ExtensionSetReference, AdminError> {
|
||||
Err(AdminError::invalid("unused plugin fixture"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,7 +247,7 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let register = source(root, "src/service/settings.rs")
|
||||
let register = source(root, "src/public_service/settings.rs")
|
||||
.items
|
||||
.into_iter()
|
||||
.find_map(|item| match item {
|
||||
@@ -293,7 +293,7 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
let generated = quote! {
|
||||
//! 宿主设置服务的操作标识与请求、响应数据合同
|
||||
//!
|
||||
//! 从宿主设置类型与 service/settings.rs 生成;更新命令见 SDK 维护说明
|
||||
//! 从宿主设置类型与 public_service/settings.rs 生成;更新命令见 SDK 维护说明
|
||||
|
||||
use super::Operation;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -28,7 +28,7 @@ struct PricingSettingsStore {
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_service_calls_freeze_empty_plan_and_cancel_before_entering_terminal() {
|
||||
use gateway_admin::service::{Origin, Plan, scope};
|
||||
use gateway_admin::public_service::{Origin, Plan, scope};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let resolutions = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -138,7 +138,7 @@ impl ApiBundle {
|
||||
}
|
||||
|
||||
/// 内部调用使用同一总路由;组合根持有强引用,Runtime 只保存 Weak
|
||||
pub fn dispatcher(&self) -> Arc<dyn gateway_core::middleware::http::Dispatcher> {
|
||||
pub fn dispatcher(&self) -> Arc<dyn gateway_core::engine::middleware::http::Dispatcher> {
|
||||
Arc::new(middleware::RouterDispatcher {
|
||||
router: middleware::wrap(
|
||||
self.router.clone(),
|
||||
|
||||
@@ -17,23 +17,21 @@ use axum::{
|
||||
http::{HeaderName, HeaderValue, Request, StatusCode},
|
||||
response::{IntoResponse as _, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::middleware::FrozenMiddlewarePlan,
|
||||
lifecycle::CancellationToken,
|
||||
middleware::{
|
||||
compose,
|
||||
http::{self as contract, Settings},
|
||||
},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::http as contract;
|
||||
use gateway_core::engine::middleware::http::Settings;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::middleware::compose;
|
||||
use http_body::{Body as HttpBody, Frame, SizeHint};
|
||||
use http_body_util::BodyExt as _;
|
||||
use tokio::time::Instant;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
tokio::task_local! { static CURRENT: contract::Context; }
|
||||
tokio::task_local! { static SETTINGS: Option<gateway_core::settings::RequestSettings>; }
|
||||
tokio::task_local! { static SETTINGS: Option<gateway_core::routing::request_settings::RequestSettings>; }
|
||||
|
||||
pub(crate) fn current_settings() -> Option<gateway_core::settings::RequestSettings> {
|
||||
pub(crate) fn current_settings() -> Option<gateway_core::routing::request_settings::RequestSettings>
|
||||
{
|
||||
SETTINGS.try_with(Clone::clone).ok().flatten()
|
||||
}
|
||||
|
||||
@@ -52,7 +50,7 @@ pub(crate) async fn scope<T>(
|
||||
}
|
||||
|
||||
pub(crate) type SettingsSource =
|
||||
Arc<dyn Fn() -> Option<gateway_core::settings::RequestSettings> + Send + Sync>;
|
||||
Arc<dyn Fn() -> Option<gateway_core::routing::request_settings::RequestSettings> + Send + Sync>;
|
||||
|
||||
pub(crate) struct RouterDispatcher {
|
||||
pub(crate) router: Router,
|
||||
@@ -60,7 +58,7 @@ pub(crate) struct RouterDispatcher {
|
||||
}
|
||||
|
||||
impl contract::Dispatcher for RouterDispatcher {
|
||||
fn request_settings(&self) -> Option<gateway_core::settings::RequestSettings> {
|
||||
fn request_settings(&self) -> Option<gateway_core::routing::request_settings::RequestSettings> {
|
||||
(self.settings)()
|
||||
}
|
||||
|
||||
@@ -195,12 +193,12 @@ async fn handle(
|
||||
});
|
||||
request.extensions_mut().insert(guard.clone());
|
||||
request.extensions_mut().insert(context.clone());
|
||||
let service_context = gateway_admin::service::Origin {
|
||||
let service_context = gateway_admin::public_service::Origin {
|
||||
request_id: context.request_id.clone(),
|
||||
call_id: context.call_id.clone(),
|
||||
cancellation: context.cancellation.clone(),
|
||||
extensions: context.extensions.clone(),
|
||||
plan: gateway_admin::service::Plan::Frozen(plan.clone()),
|
||||
plan: gateway_admin::public_service::Plan::Frozen(plan.clone()),
|
||||
};
|
||||
request
|
||||
.extensions_mut()
|
||||
@@ -220,7 +218,7 @@ async fn handle(
|
||||
terminal_context,
|
||||
SETTINGS.scope(
|
||||
settings,
|
||||
gateway_admin::service::scope(
|
||||
gateway_admin::public_service::scope(
|
||||
service_context,
|
||||
dispatch(router, request.map(Body::new), started),
|
||||
),
|
||||
|
||||
@@ -14,15 +14,14 @@ use futures::{
|
||||
future::BoxFuture,
|
||||
stream::{SplitSink, SplitStream},
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareError, MiddlewareHeader},
|
||||
lifecycle::CancellationToken,
|
||||
middleware::{
|
||||
compose,
|
||||
http::{self as http, upgrade},
|
||||
websocket as core,
|
||||
},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareError;
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_core::engine::middleware::http;
|
||||
use gateway_core::engine::middleware::http::upgrade;
|
||||
use gateway_core::engine::middleware::websocket as core;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::middleware::compose;
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
pub(super) struct Upgrade {
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
//! 下游连接的帧、写入结果、退出原因与队列预算
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
pub(super) const OUTBOUND_COMMAND_BUFFER: usize = 32;
|
||||
pub(super) const INBOUND_EVENT_BUFFER: usize = 32;
|
||||
pub(super) const DOWNSTREAM_WRITE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
|
||||
pub(super) const DOWNSTREAM_CLOSE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
pub(super) const CONNECTION_MAX_AGE: Duration = Duration::from_secs(60 * 60);
|
||||
pub(super) const DOWNSTREAM_PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
|
||||
/// WebSocket pump 的写入与生命周期预算
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ConnectionConfig {
|
||||
pub(super) write_timeout: Duration,
|
||||
pub(super) max_age: Duration,
|
||||
}
|
||||
|
||||
impl ConnectionConfig {
|
||||
/// 生产环境固定预算
|
||||
pub const PRODUCTION: Self = Self {
|
||||
write_timeout: DOWNSTREAM_WRITE_TIMEOUT,
|
||||
max_age: CONNECTION_MAX_AGE,
|
||||
};
|
||||
}
|
||||
|
||||
/// 业务层可观察的客户端输入;Ping/Pong 始终由 pump 消费
|
||||
pub enum ConnectionEvent {
|
||||
Text(String),
|
||||
Binary,
|
||||
Expired,
|
||||
Exited(PumpExitReason),
|
||||
}
|
||||
|
||||
/// 接收队列与活动期间暂存的请求共用容量;移动帧不释放它占用的名额
|
||||
pub(in super::super) struct PendingConnectionEvent {
|
||||
pub(in super::super) event: ConnectionEvent,
|
||||
pub(super) _permit: Option<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
impl PendingConnectionEvent {
|
||||
pub(super) fn exited(reason: PumpExitReason) -> Self {
|
||||
Self {
|
||||
event: ConnectionEvent::Exited(reason),
|
||||
_permit: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 下游写入阶段;名称刻意使用 write,而不是暗示客户端已消费的 delivery
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FramePhase {
|
||||
Metadata,
|
||||
First,
|
||||
Data,
|
||||
Terminal,
|
||||
FirstAndTerminal,
|
||||
Error,
|
||||
ConnectionLimit,
|
||||
Close,
|
||||
}
|
||||
|
||||
impl FramePhase {
|
||||
pub(super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Metadata => "metadata",
|
||||
Self::First => "first",
|
||||
Self::Data => "data",
|
||||
Self::Terminal => "terminal",
|
||||
Self::FirstAndTerminal => "first_and_terminal",
|
||||
Self::Error => "error",
|
||||
Self::ConnectionLimit => "connection_limit",
|
||||
Self::Close => "close",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const fn is_milestone(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::First
|
||||
| Self::Terminal
|
||||
| Self::FirstAndTerminal
|
||||
| Self::Error
|
||||
| Self::ConnectionLimit
|
||||
| Self::Close
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次下游写入的请求归属与协议阶段
|
||||
#[derive(Clone)]
|
||||
pub struct WriteContext {
|
||||
pub(super) request_id: Option<Arc<str>>,
|
||||
pub(super) phase: FramePhase,
|
||||
}
|
||||
|
||||
impl WriteContext {
|
||||
/// 创建归属于某个请求的写入上下文
|
||||
#[must_use]
|
||||
pub fn request(request_id: &Arc<str>, phase: FramePhase) -> Self {
|
||||
Self {
|
||||
request_id: Some(Arc::clone(request_id)),
|
||||
phase,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建连接级写入上下文
|
||||
#[must_use]
|
||||
pub const fn connection(phase: FramePhase) -> Self {
|
||||
Self {
|
||||
request_id: None,
|
||||
phase,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// WebSocket pump 停止的稳定原因
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PumpExitReason {
|
||||
ClientClose,
|
||||
PeerEof,
|
||||
ReadError,
|
||||
WriteError,
|
||||
WriteTimeout,
|
||||
LifecycleShutdown,
|
||||
ConnectionMaxAge,
|
||||
CoordinatorDropped,
|
||||
InboundOverload,
|
||||
ServerClose,
|
||||
PumpStopped,
|
||||
MiddlewareFailed,
|
||||
}
|
||||
|
||||
impl PumpExitReason {
|
||||
pub(in super::super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ClientClose => "client_close",
|
||||
Self::PeerEof => "peer_eof",
|
||||
Self::ReadError => "read_error",
|
||||
Self::WriteError => "write_error",
|
||||
Self::WriteTimeout => "write_timeout",
|
||||
Self::LifecycleShutdown => "lifecycle_shutdown",
|
||||
Self::ConnectionMaxAge => "connection_max_age",
|
||||
Self::CoordinatorDropped => "coordinator_dropped",
|
||||
Self::InboundOverload => "inbound_overload",
|
||||
Self::ServerClose => "server_close",
|
||||
Self::PumpStopped => "pump_stopped",
|
||||
Self::MiddlewareFailed => "middleware_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 下游 WebSocket 写入失败
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ConnectionWriteError {
|
||||
#[error("downstream WebSocket pump is closed")]
|
||||
Closed,
|
||||
#[error("downstream WebSocket write timed out after {timeout:?}")]
|
||||
Timeout { timeout: Duration },
|
||||
#[error("downstream WebSocket transport write failed: {message}")]
|
||||
Transport { message: String },
|
||||
}
|
||||
|
||||
/// 只有实际 transport 写入才记为 Written;插件丢弃消息不伪造写入成功
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum WriteOutcome {
|
||||
Written,
|
||||
Suppressed,
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
//! 协调层持有的连接句柄与唯一 pump 启动入口
|
||||
|
||||
use super::{
|
||||
frame::*,
|
||||
pump::run_pump,
|
||||
state::{ConnectionCommand, ConnectionStats},
|
||||
};
|
||||
use axum::extract::ws::{CloseFrame, Message, WebSocket, close_code};
|
||||
use futures::{Sink, Stream};
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareHeader},
|
||||
lifecycle::CancellationToken,
|
||||
};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
fmt,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
sync::{mpsc, oneshot},
|
||||
task::JoinHandle,
|
||||
time::{Instant, timeout},
|
||||
};
|
||||
|
||||
/// 协调层持有的单 owner WebSocket 连接句柄
|
||||
pub struct ResponsesWebSocketConnection {
|
||||
connection_id: Arc<str>,
|
||||
opened_at: Instant,
|
||||
expired: Arc<AtomicBool>,
|
||||
commands: Option<mpsc::Sender<ConnectionCommand>>,
|
||||
incoming: mpsc::Receiver<PendingConnectionEvent>,
|
||||
deferred: VecDeque<PendingConnectionEvent>,
|
||||
exited: oneshot::Receiver<PumpExitReason>,
|
||||
pump_task: Option<JoinHandle<()>>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
config: ConnectionConfig,
|
||||
exit_reason: Option<PumpExitReason>,
|
||||
}
|
||||
|
||||
impl ResponsesWebSocketConnection {
|
||||
pub(in super::super) fn new(
|
||||
socket: WebSocket,
|
||||
connection_id: String,
|
||||
cancellation: CancellationToken,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) -> Self {
|
||||
spawn_connection(
|
||||
socket,
|
||||
Arc::<str>::from(connection_id),
|
||||
cancellation,
|
||||
ConnectionConfig::PRODUCTION,
|
||||
middleware,
|
||||
headers,
|
||||
)
|
||||
}
|
||||
|
||||
pub(in super::super) fn id(&self) -> &str {
|
||||
&self.connection_id
|
||||
}
|
||||
|
||||
/// 返回连接是否已经达到生命周期上限
|
||||
#[must_use]
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.expired.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(in super::super) fn age(&self) -> Duration {
|
||||
self.opened_at.elapsed()
|
||||
}
|
||||
|
||||
/// 等待下一个需要业务层处理的客户端事件
|
||||
pub async fn next_event(&mut self) -> Option<ConnectionEvent> {
|
||||
if self.exit_reason.is_none() {
|
||||
match self.exited.try_recv() {
|
||||
Ok(reason) => {
|
||||
self.exit_reason = Some(reason);
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
|
||||
self.exit_reason = Some(PumpExitReason::PumpStopped);
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {}
|
||||
}
|
||||
}
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return Some(ConnectionEvent::Exited(reason));
|
||||
}
|
||||
if let Some(event) = self.deferred.pop_front() {
|
||||
return Some(event.event);
|
||||
}
|
||||
self.next_active_event().await.map(|event| event.event)
|
||||
}
|
||||
|
||||
/// 活动响应期间只消费新到的帧,已排队的下一轮请求继续保持串行
|
||||
pub(in super::super) async fn next_active_event(&mut self) -> Option<PendingConnectionEvent> {
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return Some(PendingConnectionEvent::exited(reason));
|
||||
}
|
||||
let event = tokio::select! {
|
||||
biased;
|
||||
reason = &mut self.exited => {
|
||||
Some(PendingConnectionEvent::exited(reason.unwrap_or(PumpExitReason::PumpStopped)))
|
||||
}
|
||||
event = self.incoming.recv() => event,
|
||||
};
|
||||
if let Some(PendingConnectionEvent {
|
||||
event: ConnectionEvent::Exited(reason),
|
||||
..
|
||||
}) = event.as_ref()
|
||||
{
|
||||
self.exit_reason.get_or_insert(*reason);
|
||||
} else if event.is_none() {
|
||||
self.exit_reason.get_or_insert(PumpExitReason::PumpStopped);
|
||||
}
|
||||
event
|
||||
}
|
||||
|
||||
pub(in super::super) fn defer(&mut self, event: PendingConnectionEvent) {
|
||||
self.deferred.push_back(event);
|
||||
}
|
||||
|
||||
/// 等待连接退出,不消费留给后续串行请求的业务帧
|
||||
pub async fn wait_for_exit(&mut self) -> PumpExitReason {
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return reason;
|
||||
}
|
||||
let reason = (&mut self.exited)
|
||||
.await
|
||||
.unwrap_or(PumpExitReason::PumpStopped);
|
||||
self.exit_reason = Some(reason);
|
||||
reason
|
||||
}
|
||||
|
||||
/// 串行写入文本帧,并等待 pump 确认 transport 写入结果
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// pump 已关闭、写入超时或 transport 失败时返回稳定错误
|
||||
pub async fn send_text(
|
||||
&mut self,
|
||||
payload: String,
|
||||
context: WriteContext,
|
||||
) -> Result<WriteOutcome, ConnectionWriteError> {
|
||||
self.send(Message::Text(payload.into()), context).await
|
||||
}
|
||||
|
||||
pub(in super::super) async fn close_policy(
|
||||
&mut self,
|
||||
reason: &'static str,
|
||||
request_id: Option<&Arc<str>>,
|
||||
) {
|
||||
let context = request_id.map_or_else(
|
||||
|| WriteContext::connection(FramePhase::Close),
|
||||
|request_id| WriteContext::request(request_id, FramePhase::Close),
|
||||
);
|
||||
self.close(
|
||||
CloseFrame {
|
||||
code: close_code::POLICY,
|
||||
reason: reason.into(),
|
||||
},
|
||||
context,
|
||||
PumpExitReason::ServerClose,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(in super::super) async fn close_for_connection_limit(&mut self, reason: &'static str) {
|
||||
self.close(
|
||||
CloseFrame {
|
||||
code: close_code::NORMAL,
|
||||
reason: reason.into(),
|
||||
},
|
||||
WriteContext::connection(FramePhase::Close),
|
||||
PumpExitReason::ConnectionMaxAge,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn close(
|
||||
&mut self,
|
||||
frame: CloseFrame,
|
||||
context: WriteContext,
|
||||
exit_reason: PumpExitReason,
|
||||
) {
|
||||
if self
|
||||
.send(Message::Close(Some(frame)), context)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.exit_reason.get_or_insert(exit_reason);
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(
|
||||
&mut self,
|
||||
message: Message,
|
||||
context: WriteContext,
|
||||
) -> Result<WriteOutcome, ConnectionWriteError> {
|
||||
let Some(commands) = self.commands.as_ref().cloned() else {
|
||||
return Err(ConnectionWriteError::Closed);
|
||||
};
|
||||
self.stats.observe_command_queue(&commands);
|
||||
let (acknowledged, acknowledgement) = oneshot::channel();
|
||||
let started_at = Instant::now();
|
||||
let request_id = context.request_id.clone();
|
||||
let phase = context.phase;
|
||||
let command = ConnectionCommand {
|
||||
message,
|
||||
context,
|
||||
acknowledged,
|
||||
};
|
||||
let write = async move {
|
||||
commands
|
||||
.send(command)
|
||||
.await
|
||||
.map_err(|_| ConnectionWriteError::Closed)?;
|
||||
acknowledgement
|
||||
.await
|
||||
.map_err(|_| ConnectionWriteError::Closed)?
|
||||
};
|
||||
let result = match timeout(self.config.write_timeout, write).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(ConnectionWriteError::Timeout {
|
||||
timeout: self.config.write_timeout,
|
||||
}),
|
||||
};
|
||||
let duration = started_at.elapsed();
|
||||
match &result {
|
||||
Ok(WriteOutcome::Written) if phase.is_milestone() => tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
"Responses WebSocket frame write succeeded"
|
||||
),
|
||||
Ok(outcome) => tracing::debug!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
?outcome,
|
||||
"Responses WebSocket frame processed"
|
||||
),
|
||||
Err(error) => tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
error = %error,
|
||||
"Responses WebSocket frame write failed"
|
||||
),
|
||||
}
|
||||
if let Err(error) = &result {
|
||||
let reason = match error {
|
||||
ConnectionWriteError::Timeout { .. } => PumpExitReason::WriteTimeout,
|
||||
ConnectionWriteError::Transport { .. } => PumpExitReason::WriteError,
|
||||
ConnectionWriteError::Closed => PumpExitReason::PumpStopped,
|
||||
};
|
||||
self.terminate(reason);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn terminate(&mut self, reason: PumpExitReason) {
|
||||
self.exit_reason.get_or_insert(reason);
|
||||
self.commands.take();
|
||||
if let Some(task) = self.pump_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(in super::super) fn log_summary(&self, request_count: u64) {
|
||||
let connection_age_ms = self.age().as_millis();
|
||||
let read_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_read_ms.load(Ordering::Relaxed)));
|
||||
let write_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_write_ms.load(Ordering::Relaxed)));
|
||||
tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_count,
|
||||
connection_age_ms,
|
||||
pump_exit_reason = self
|
||||
.exit_reason
|
||||
.map_or("unknown", PumpExitReason::as_str),
|
||||
command_queue_high_water = self
|
||||
.stats
|
||||
.command_queue_high_water
|
||||
.load(Ordering::Relaxed),
|
||||
command_backpressure_count = self
|
||||
.stats
|
||||
.command_backpressure_count
|
||||
.load(Ordering::Relaxed),
|
||||
ping_received_count = self.stats.ping_received_count.load(Ordering::Relaxed),
|
||||
ping_written_count = self.stats.ping_written_count.load(Ordering::Relaxed),
|
||||
pong_received_count = self.stats.pong_received_count.load(Ordering::Relaxed),
|
||||
read_idle_ms,
|
||||
write_idle_ms,
|
||||
"Responses WebSocket disconnected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponsesWebSocketConnection {
|
||||
fn drop(&mut self) {
|
||||
self.commands.take();
|
||||
if let Some(task) = self.pump_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 WebSocket transport 启动包含冻结中间件计划的单 owner pump
|
||||
pub fn spawn_connection<S, E>(
|
||||
socket: S,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
config: ConnectionConfig,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) -> ResponsesWebSocketConnection
|
||||
where
|
||||
S: Stream<Item = Result<Message, E>> + Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let opened_at = Instant::now();
|
||||
let expired = Arc::new(AtomicBool::new(false));
|
||||
let stats = Arc::new(ConnectionStats::default());
|
||||
let (command_tx, command_rx) = mpsc::channel(OUTBOUND_COMMAND_BUFFER);
|
||||
let (incoming_tx, incoming_rx) = mpsc::channel(INBOUND_EVENT_BUFFER);
|
||||
let (exit_tx, exit_rx) = oneshot::channel();
|
||||
let pump_task = tokio::spawn(run_pump(
|
||||
socket,
|
||||
command_rx,
|
||||
incoming_tx,
|
||||
exit_tx,
|
||||
Arc::clone(&connection_id),
|
||||
cancellation,
|
||||
opened_at,
|
||||
Arc::clone(&expired),
|
||||
Arc::clone(&stats),
|
||||
config,
|
||||
middleware,
|
||||
headers,
|
||||
));
|
||||
ResponsesWebSocketConnection {
|
||||
connection_id,
|
||||
opened_at,
|
||||
expired,
|
||||
commands: Some(command_tx),
|
||||
incoming: incoming_rx,
|
||||
deferred: VecDeque::new(),
|
||||
exited: exit_rx,
|
||||
pump_task: Some(pump_task),
|
||||
stats,
|
||||
config,
|
||||
exit_reason: None,
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
//! 插件消息组合在 transport 锁之外执行,主动发送复用唯一 writer
|
||||
use super::{ConnectionWriteError, PumpContext, write_message};
|
||||
use super::{ConnectionWriteError, frame::DOWNSTREAM_CLOSE_TIMEOUT, state::PumpContext};
|
||||
pub(super) use crate::middleware::websocket::CancelOnDrop;
|
||||
use crate::middleware::websocket::{from_core, into_core};
|
||||
use axum::extract::ws::Message;
|
||||
use futures::{Sink, future::BoxFuture};
|
||||
use gateway_core::{engine::middleware::MiddlewareError, middleware::websocket as core};
|
||||
use std::sync::Arc;
|
||||
use futures::{Sink, SinkExt, future::BoxFuture};
|
||||
use gateway_core::engine::middleware::{MiddlewareError, websocket as core};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use std::{fmt, sync::Arc};
|
||||
use tokio::time::timeout;
|
||||
|
||||
pub(super) struct Writer<S> {
|
||||
socket: tokio::sync::Mutex<S>,
|
||||
@@ -37,6 +39,36 @@ where
|
||||
}) => result.unwrap_or(Err(ConnectionWriteError::Timeout { timeout: self.context.config.write_timeout })),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn finish_peer_close(
|
||||
&self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), ConnectionWriteError> {
|
||||
// 主动 Sender 也可能在 pump 之外持锁;先独占 transport 再取消业务,
|
||||
// 有在途写或已取消的半帧时立即跳过,不能等待它释放锁后继续刷新
|
||||
let socket = self
|
||||
.socket
|
||||
.try_lock()
|
||||
.ok()
|
||||
.filter(|_| !self.context.cancellation.is_cancelled());
|
||||
self.context.cancellation.cancel();
|
||||
let Some(mut socket) = socket else {
|
||||
return Err(ConnectionWriteError::Closed);
|
||||
};
|
||||
// 收到 Close 后 transport 已排队应答;只刷新它,不能再发送一份 Close
|
||||
// 业务调用已取消,清理只受宿主取消与独立短预算约束,不沿用业务写入的长超时
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = cancellation.cancelled() => Err(ConnectionWriteError::Closed),
|
||||
result = timeout(DOWNSTREAM_CLOSE_TIMEOUT, async {
|
||||
socket.flush().await.map_err(|error| {
|
||||
ConnectionWriteError::Transport { message: error.to_string() }
|
||||
})
|
||||
}) => result.unwrap_or(Err(ConnectionWriteError::Timeout {
|
||||
timeout: DOWNSTREAM_CLOSE_TIMEOUT,
|
||||
})),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<S, E> core::Sender for Writer<S>
|
||||
where
|
||||
@@ -81,3 +113,25 @@ pub(super) async fn transform(
|
||||
.map(from_core)
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn write_message<S, E>(
|
||||
socket: &mut S,
|
||||
message: Message,
|
||||
context: &PumpContext,
|
||||
) -> Result<(), ConnectionWriteError>
|
||||
where
|
||||
S: Sink<Message, Error = E> + Unpin,
|
||||
E: fmt::Display,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => Err(ConnectionWriteError::Closed),
|
||||
result = timeout(context.config.write_timeout, socket.send(message)) => {
|
||||
match result {
|
||||
Ok(Ok(())) => { context.stats.record_write(context.opened_at); Ok(()) }
|
||||
Ok(Err(error)) => Err(ConnectionWriteError::Transport { message: error.to_string() }),
|
||||
Err(_) => Err(ConnectionWriteError::Timeout { timeout: context.config.write_timeout }),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,806 +1,13 @@
|
||||
//! 下游客户端 WebSocket 的单 owner pump 与有界收发边界
|
||||
//! 下游客户端 WebSocket 的连接合同、句柄与单 owner pump
|
||||
|
||||
mod frame;
|
||||
mod handle;
|
||||
mod middleware;
|
||||
mod pump;
|
||||
mod state;
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
fmt,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
pub use frame::{
|
||||
ConnectionConfig, ConnectionEvent, ConnectionWriteError, FramePhase, PumpExitReason,
|
||||
WriteContext, WriteOutcome,
|
||||
};
|
||||
|
||||
use axum::extract::ws::{CloseFrame, Message, WebSocket, close_code};
|
||||
use futures::{Sink, SinkExt, Stream, StreamExt};
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareHeader},
|
||||
lifecycle::CancellationToken,
|
||||
middleware::websocket as contract,
|
||||
};
|
||||
use thiserror::Error;
|
||||
use tokio::{
|
||||
sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot},
|
||||
task::JoinHandle,
|
||||
time::{Instant, timeout},
|
||||
};
|
||||
|
||||
const OUTBOUND_COMMAND_BUFFER: usize = 32;
|
||||
const INBOUND_EVENT_BUFFER: usize = 32;
|
||||
const DOWNSTREAM_WRITE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
|
||||
const CONNECTION_MAX_AGE: Duration = Duration::from_secs(60 * 60);
|
||||
const DOWNSTREAM_PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
|
||||
/// WebSocket pump 的写入与生命周期预算
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ConnectionConfig {
|
||||
write_timeout: Duration,
|
||||
max_age: Duration,
|
||||
}
|
||||
|
||||
impl ConnectionConfig {
|
||||
/// 生产环境固定预算
|
||||
pub const PRODUCTION: Self = Self {
|
||||
write_timeout: DOWNSTREAM_WRITE_TIMEOUT,
|
||||
max_age: CONNECTION_MAX_AGE,
|
||||
};
|
||||
}
|
||||
|
||||
/// 业务层可观察的客户端输入;Ping/Pong 始终由 pump 消费
|
||||
pub enum ConnectionEvent {
|
||||
Text(String),
|
||||
Binary,
|
||||
Expired,
|
||||
Exited(PumpExitReason),
|
||||
}
|
||||
|
||||
/// 接收队列与活动期间暂存的请求共用容量;移动帧不释放它占用的名额
|
||||
pub(super) struct PendingConnectionEvent {
|
||||
pub(super) event: ConnectionEvent,
|
||||
_permit: Option<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
impl PendingConnectionEvent {
|
||||
fn exited(reason: PumpExitReason) -> Self {
|
||||
Self {
|
||||
event: ConnectionEvent::Exited(reason),
|
||||
_permit: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 下游写入阶段;名称刻意使用 write,而不是暗示客户端已消费的 delivery
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FramePhase {
|
||||
Metadata,
|
||||
First,
|
||||
Data,
|
||||
Terminal,
|
||||
FirstAndTerminal,
|
||||
Error,
|
||||
ConnectionLimit,
|
||||
Close,
|
||||
}
|
||||
|
||||
impl FramePhase {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Metadata => "metadata",
|
||||
Self::First => "first",
|
||||
Self::Data => "data",
|
||||
Self::Terminal => "terminal",
|
||||
Self::FirstAndTerminal => "first_and_terminal",
|
||||
Self::Error => "error",
|
||||
Self::ConnectionLimit => "connection_limit",
|
||||
Self::Close => "close",
|
||||
}
|
||||
}
|
||||
|
||||
const fn is_milestone(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::First
|
||||
| Self::Terminal
|
||||
| Self::FirstAndTerminal
|
||||
| Self::Error
|
||||
| Self::ConnectionLimit
|
||||
| Self::Close
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次下游写入的请求归属与协议阶段
|
||||
#[derive(Clone)]
|
||||
pub struct WriteContext {
|
||||
request_id: Option<Arc<str>>,
|
||||
phase: FramePhase,
|
||||
}
|
||||
|
||||
impl WriteContext {
|
||||
/// 创建归属于某个请求的写入上下文
|
||||
#[must_use]
|
||||
pub fn request(request_id: &Arc<str>, phase: FramePhase) -> Self {
|
||||
Self {
|
||||
request_id: Some(Arc::clone(request_id)),
|
||||
phase,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建连接级写入上下文
|
||||
#[must_use]
|
||||
pub const fn connection(phase: FramePhase) -> Self {
|
||||
Self {
|
||||
request_id: None,
|
||||
phase,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// WebSocket pump 停止的稳定原因
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PumpExitReason {
|
||||
ClientClose,
|
||||
PeerEof,
|
||||
ReadError,
|
||||
WriteError,
|
||||
WriteTimeout,
|
||||
LifecycleShutdown,
|
||||
ConnectionMaxAge,
|
||||
CoordinatorDropped,
|
||||
InboundOverload,
|
||||
ServerClose,
|
||||
PumpStopped,
|
||||
MiddlewareFailed,
|
||||
}
|
||||
|
||||
impl PumpExitReason {
|
||||
pub(super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ClientClose => "client_close",
|
||||
Self::PeerEof => "peer_eof",
|
||||
Self::ReadError => "read_error",
|
||||
Self::WriteError => "write_error",
|
||||
Self::WriteTimeout => "write_timeout",
|
||||
Self::LifecycleShutdown => "lifecycle_shutdown",
|
||||
Self::ConnectionMaxAge => "connection_max_age",
|
||||
Self::CoordinatorDropped => "coordinator_dropped",
|
||||
Self::InboundOverload => "inbound_overload",
|
||||
Self::ServerClose => "server_close",
|
||||
Self::PumpStopped => "pump_stopped",
|
||||
Self::MiddlewareFailed => "middleware_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 下游 WebSocket 写入失败
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ConnectionWriteError {
|
||||
#[error("downstream WebSocket pump is closed")]
|
||||
Closed,
|
||||
#[error("downstream WebSocket write timed out after {timeout:?}")]
|
||||
Timeout { timeout: Duration },
|
||||
#[error("downstream WebSocket transport write failed: {message}")]
|
||||
Transport { message: String },
|
||||
}
|
||||
|
||||
/// 只有实际 transport 写入才记为 Written;插件丢弃消息不伪造写入成功
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum WriteOutcome {
|
||||
Written,
|
||||
Suppressed,
|
||||
}
|
||||
|
||||
struct ConnectionCommand {
|
||||
message: Message,
|
||||
context: WriteContext,
|
||||
acknowledged: oneshot::Sender<Result<WriteOutcome, ConnectionWriteError>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ConnectionStats {
|
||||
command_queue_high_water: AtomicUsize,
|
||||
command_backpressure_count: AtomicU64,
|
||||
ping_received_count: AtomicU64,
|
||||
ping_written_count: AtomicU64,
|
||||
pong_received_count: AtomicU64,
|
||||
last_read_ms: AtomicU64,
|
||||
last_write_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl ConnectionStats {
|
||||
fn record_read(&self, opened_at: Instant) {
|
||||
self.last_read_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
fn record_write(&self, opened_at: Instant) {
|
||||
self.last_write_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
fn observe_command_queue(&self, sender: &mpsc::Sender<ConnectionCommand>) {
|
||||
let queued = OUTBOUND_COMMAND_BUFFER
|
||||
.saturating_sub(sender.capacity())
|
||||
.saturating_add(1)
|
||||
.min(OUTBOUND_COMMAND_BUFFER);
|
||||
self.command_queue_high_water
|
||||
.fetch_max(queued, Ordering::Relaxed);
|
||||
if sender.capacity() == 0 {
|
||||
self.command_backpressure_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 协调层持有的单 owner WebSocket 连接句柄
|
||||
pub struct ResponsesWebSocketConnection {
|
||||
connection_id: Arc<str>,
|
||||
opened_at: Instant,
|
||||
expired: Arc<AtomicBool>,
|
||||
commands: Option<mpsc::Sender<ConnectionCommand>>,
|
||||
incoming: mpsc::Receiver<PendingConnectionEvent>,
|
||||
deferred: VecDeque<PendingConnectionEvent>,
|
||||
exited: oneshot::Receiver<PumpExitReason>,
|
||||
pump_task: Option<JoinHandle<()>>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
config: ConnectionConfig,
|
||||
exit_reason: Option<PumpExitReason>,
|
||||
}
|
||||
|
||||
impl ResponsesWebSocketConnection {
|
||||
pub(super) fn new(
|
||||
socket: WebSocket,
|
||||
connection_id: String,
|
||||
cancellation: CancellationToken,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) -> Self {
|
||||
spawn_connection(
|
||||
socket,
|
||||
Arc::<str>::from(connection_id),
|
||||
cancellation,
|
||||
ConnectionConfig::PRODUCTION,
|
||||
middleware,
|
||||
headers,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn id(&self) -> &str {
|
||||
&self.connection_id
|
||||
}
|
||||
|
||||
/// 返回连接是否已经达到生命周期上限
|
||||
#[must_use]
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.expired.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(super) fn age(&self) -> Duration {
|
||||
self.opened_at.elapsed()
|
||||
}
|
||||
|
||||
/// 等待下一个需要业务层处理的客户端事件
|
||||
pub async fn next_event(&mut self) -> Option<ConnectionEvent> {
|
||||
if self.exit_reason.is_none() {
|
||||
match self.exited.try_recv() {
|
||||
Ok(reason) => {
|
||||
self.exit_reason = Some(reason);
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
|
||||
self.exit_reason = Some(PumpExitReason::PumpStopped);
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {}
|
||||
}
|
||||
}
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return Some(ConnectionEvent::Exited(reason));
|
||||
}
|
||||
if let Some(event) = self.deferred.pop_front() {
|
||||
return Some(event.event);
|
||||
}
|
||||
self.next_active_event().await.map(|event| event.event)
|
||||
}
|
||||
|
||||
/// 活动响应期间只消费新到的帧,已排队的下一轮请求继续保持串行
|
||||
pub(super) async fn next_active_event(&mut self) -> Option<PendingConnectionEvent> {
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return Some(PendingConnectionEvent::exited(reason));
|
||||
}
|
||||
let event = tokio::select! {
|
||||
biased;
|
||||
reason = &mut self.exited => {
|
||||
Some(PendingConnectionEvent::exited(reason.unwrap_or(PumpExitReason::PumpStopped)))
|
||||
}
|
||||
event = self.incoming.recv() => event,
|
||||
};
|
||||
if let Some(PendingConnectionEvent {
|
||||
event: ConnectionEvent::Exited(reason),
|
||||
..
|
||||
}) = event.as_ref()
|
||||
{
|
||||
self.exit_reason.get_or_insert(*reason);
|
||||
} else if event.is_none() {
|
||||
self.exit_reason.get_or_insert(PumpExitReason::PumpStopped);
|
||||
}
|
||||
event
|
||||
}
|
||||
|
||||
pub(super) fn defer(&mut self, event: PendingConnectionEvent) {
|
||||
self.deferred.push_back(event);
|
||||
}
|
||||
|
||||
/// 等待连接退出,不消费留给后续串行请求的业务帧
|
||||
pub async fn wait_for_exit(&mut self) -> PumpExitReason {
|
||||
if let Some(reason) = self.exit_reason {
|
||||
return reason;
|
||||
}
|
||||
let reason = (&mut self.exited)
|
||||
.await
|
||||
.unwrap_or(PumpExitReason::PumpStopped);
|
||||
self.exit_reason = Some(reason);
|
||||
reason
|
||||
}
|
||||
|
||||
/// 串行写入文本帧,并等待 pump 确认 transport 写入结果
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// pump 已关闭、写入超时或 transport 失败时返回稳定错误
|
||||
pub async fn send_text(
|
||||
&mut self,
|
||||
payload: String,
|
||||
context: WriteContext,
|
||||
) -> Result<WriteOutcome, ConnectionWriteError> {
|
||||
self.send(Message::Text(payload.into()), context).await
|
||||
}
|
||||
|
||||
pub(super) async fn close_policy(
|
||||
&mut self,
|
||||
reason: &'static str,
|
||||
request_id: Option<&Arc<str>>,
|
||||
) {
|
||||
let context = request_id.map_or_else(
|
||||
|| WriteContext::connection(FramePhase::Close),
|
||||
|request_id| WriteContext::request(request_id, FramePhase::Close),
|
||||
);
|
||||
self.close(
|
||||
CloseFrame {
|
||||
code: close_code::POLICY,
|
||||
reason: reason.into(),
|
||||
},
|
||||
context,
|
||||
PumpExitReason::ServerClose,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn close_for_connection_limit(&mut self, reason: &'static str) {
|
||||
self.close(
|
||||
CloseFrame {
|
||||
code: close_code::NORMAL,
|
||||
reason: reason.into(),
|
||||
},
|
||||
WriteContext::connection(FramePhase::Close),
|
||||
PumpExitReason::ConnectionMaxAge,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn close(
|
||||
&mut self,
|
||||
frame: CloseFrame,
|
||||
context: WriteContext,
|
||||
exit_reason: PumpExitReason,
|
||||
) {
|
||||
if self
|
||||
.send(Message::Close(Some(frame)), context)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.exit_reason.get_or_insert(exit_reason);
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(
|
||||
&mut self,
|
||||
message: Message,
|
||||
context: WriteContext,
|
||||
) -> Result<WriteOutcome, ConnectionWriteError> {
|
||||
let Some(commands) = self.commands.as_ref().cloned() else {
|
||||
return Err(ConnectionWriteError::Closed);
|
||||
};
|
||||
self.stats.observe_command_queue(&commands);
|
||||
let (acknowledged, acknowledgement) = oneshot::channel();
|
||||
let started_at = Instant::now();
|
||||
let request_id = context.request_id.clone();
|
||||
let phase = context.phase;
|
||||
let command = ConnectionCommand {
|
||||
message,
|
||||
context,
|
||||
acknowledged,
|
||||
};
|
||||
let write = async move {
|
||||
commands
|
||||
.send(command)
|
||||
.await
|
||||
.map_err(|_| ConnectionWriteError::Closed)?;
|
||||
acknowledgement
|
||||
.await
|
||||
.map_err(|_| ConnectionWriteError::Closed)?
|
||||
};
|
||||
let result = match timeout(self.config.write_timeout, write).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(ConnectionWriteError::Timeout {
|
||||
timeout: self.config.write_timeout,
|
||||
}),
|
||||
};
|
||||
let duration = started_at.elapsed();
|
||||
match &result {
|
||||
Ok(WriteOutcome::Written) if phase.is_milestone() => tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
"Responses WebSocket frame write succeeded"
|
||||
),
|
||||
Ok(outcome) => tracing::debug!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
?outcome,
|
||||
"Responses WebSocket frame processed"
|
||||
),
|
||||
Err(error) => tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
frame_phase = phase.as_str(),
|
||||
write_duration_ms = duration.as_millis(),
|
||||
error = %error,
|
||||
"Responses WebSocket frame write failed"
|
||||
),
|
||||
}
|
||||
if let Err(error) = &result {
|
||||
let reason = match error {
|
||||
ConnectionWriteError::Timeout { .. } => PumpExitReason::WriteTimeout,
|
||||
ConnectionWriteError::Transport { .. } => PumpExitReason::WriteError,
|
||||
ConnectionWriteError::Closed => PumpExitReason::PumpStopped,
|
||||
};
|
||||
self.terminate(reason);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn terminate(&mut self, reason: PumpExitReason) {
|
||||
self.exit_reason.get_or_insert(reason);
|
||||
self.commands.take();
|
||||
if let Some(task) = self.pump_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn log_summary(&self, request_count: u64) {
|
||||
let connection_age_ms = self.age().as_millis();
|
||||
let read_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_read_ms.load(Ordering::Relaxed)));
|
||||
let write_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_write_ms.load(Ordering::Relaxed)));
|
||||
tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_count,
|
||||
connection_age_ms,
|
||||
pump_exit_reason = self
|
||||
.exit_reason
|
||||
.map_or("unknown", PumpExitReason::as_str),
|
||||
command_queue_high_water = self
|
||||
.stats
|
||||
.command_queue_high_water
|
||||
.load(Ordering::Relaxed),
|
||||
command_backpressure_count = self
|
||||
.stats
|
||||
.command_backpressure_count
|
||||
.load(Ordering::Relaxed),
|
||||
ping_received_count = self.stats.ping_received_count.load(Ordering::Relaxed),
|
||||
ping_written_count = self.stats.ping_written_count.load(Ordering::Relaxed),
|
||||
pong_received_count = self.stats.pong_received_count.load(Ordering::Relaxed),
|
||||
read_idle_ms,
|
||||
write_idle_ms,
|
||||
"Responses WebSocket disconnected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponsesWebSocketConnection {
|
||||
fn drop(&mut self) {
|
||||
self.commands.take();
|
||||
if let Some(task) = self.pump_task.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 为 WebSocket transport 启动包含冻结中间件计划的单 owner pump
|
||||
pub fn spawn_connection<S, E>(
|
||||
socket: S,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
config: ConnectionConfig,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) -> ResponsesWebSocketConnection
|
||||
where
|
||||
S: Stream<Item = Result<Message, E>> + Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let opened_at = Instant::now();
|
||||
let expired = Arc::new(AtomicBool::new(false));
|
||||
let stats = Arc::new(ConnectionStats::default());
|
||||
let (command_tx, command_rx) = mpsc::channel(OUTBOUND_COMMAND_BUFFER);
|
||||
let (incoming_tx, incoming_rx) = mpsc::channel(INBOUND_EVENT_BUFFER);
|
||||
let (exit_tx, exit_rx) = oneshot::channel();
|
||||
let pump_task = tokio::spawn(run_pump(
|
||||
socket,
|
||||
command_rx,
|
||||
incoming_tx,
|
||||
exit_tx,
|
||||
Arc::clone(&connection_id),
|
||||
cancellation,
|
||||
opened_at,
|
||||
Arc::clone(&expired),
|
||||
Arc::clone(&stats),
|
||||
config,
|
||||
middleware,
|
||||
headers,
|
||||
));
|
||||
ResponsesWebSocketConnection {
|
||||
connection_id,
|
||||
opened_at,
|
||||
expired,
|
||||
commands: Some(command_tx),
|
||||
incoming: incoming_rx,
|
||||
deferred: VecDeque::new(),
|
||||
exited: exit_rx,
|
||||
pump_task: Some(pump_task),
|
||||
stats,
|
||||
config,
|
||||
exit_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
async fn run_pump<S, E>(
|
||||
socket: S,
|
||||
commands: mpsc::Receiver<ConnectionCommand>,
|
||||
incoming: mpsc::Sender<PendingConnectionEvent>,
|
||||
exited: oneshot::Sender<PumpExitReason>,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
opened_at: Instant,
|
||||
expired: Arc<AtomicBool>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
config: ConnectionConfig,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) where
|
||||
S: Stream<Item = Result<Message, E>> + Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let context = Arc::new(PumpContext {
|
||||
connection_id,
|
||||
cancellation: cancellation.child_token(),
|
||||
opened_at,
|
||||
expired,
|
||||
stats,
|
||||
config,
|
||||
middleware,
|
||||
headers,
|
||||
});
|
||||
let _lifetime = middleware::CancelOnDrop(Some(context.cancellation.clone()));
|
||||
let (writer, mut reader) = socket.split();
|
||||
let writer = Arc::new(middleware::Writer::new(writer, context.clone()));
|
||||
let sender: Arc<dyn contract::Sender> = writer.clone();
|
||||
// 同一个受管任务同时驱动两侧;等待写入不能阻塞入站控制、关闭或取消
|
||||
let reason = tokio::select! {
|
||||
biased;
|
||||
reason = read_connection(&mut reader, &incoming, sender.clone(), &context) => reason,
|
||||
reason = write_connection(writer.clone(), commands, &context) => reason,
|
||||
};
|
||||
context.cancellation.cancel();
|
||||
let _ = exited.send(reason);
|
||||
tracing::debug!(websocket_connection_id = %context.connection_id, pump_exit_reason = reason.as_str(), "Responses WebSocket pump exited");
|
||||
}
|
||||
|
||||
struct PumpContext {
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
opened_at: Instant,
|
||||
expired: Arc<AtomicBool>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
config: ConnectionConfig,
|
||||
}
|
||||
|
||||
async fn read_connection<S, E>(
|
||||
socket: &mut S,
|
||||
incoming: &mpsc::Sender<PendingConnectionEvent>,
|
||||
sender: Arc<dyn contract::Sender>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
S: Stream<Item = Result<Message, E>> + Unpin,
|
||||
E: fmt::Display,
|
||||
{
|
||||
let event_slots = Arc::new(Semaphore::new(INBOUND_EVENT_BUFFER));
|
||||
let deadline = tokio::time::sleep_until(context.opened_at + context.config.max_age);
|
||||
tokio::pin!(deadline);
|
||||
let mut deadline_elapsed = false;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => return PumpExitReason::LifecycleShutdown,
|
||||
() = &mut deadline, if !deadline_elapsed => {
|
||||
deadline_elapsed = true;
|
||||
context.expired.store(true, Ordering::Release);
|
||||
// 到期只阻止下一轮,当前响应仍由协调层按原有合同收尾
|
||||
match emit_incoming(incoming, &event_slots, ConnectionEvent::Expired) {
|
||||
Ok(()) | Err(PumpExitReason::InboundOverload) => {}
|
||||
Err(reason) => return reason,
|
||||
}
|
||||
}
|
||||
message = socket.next() => {
|
||||
if matches!(&message, Some(Ok(_))) { context.stats.record_read(context.opened_at); }
|
||||
let message = match message {
|
||||
Some(Ok(message)) => match middleware::transform(message, contract::Direction::Incoming, sender.clone(), context).await {
|
||||
Ok(Some(message)) => Some(Ok(message)),
|
||||
Ok(None) => continue,
|
||||
Err(_) => return PumpExitReason::MiddlewareFailed,
|
||||
},
|
||||
other => other,
|
||||
};
|
||||
let event = match message {
|
||||
Some(Ok(Message::Text(payload))) => Some(ConnectionEvent::Text(payload.to_string())),
|
||||
Some(Ok(Message::Binary(_))) => Some(ConnectionEvent::Binary),
|
||||
Some(Ok(Message::Ping(_))) => {
|
||||
// Axum/tungstenite 在继续读取时自动刷新 Pong,不能再手工发送一份
|
||||
context.stats.ping_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
Some(Ok(Message::Pong(_))) => {
|
||||
context.stats.pong_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
Some(Ok(Message::Close(_))) => return PumpExitReason::ClientClose,
|
||||
Some(Err(error)) => {
|
||||
tracing::info!(websocket_connection_id = %context.connection_id, error = %error, "Responses WebSocket receive failed");
|
||||
return PumpExitReason::ReadError;
|
||||
}
|
||||
None => return PumpExitReason::PeerEof,
|
||||
};
|
||||
if let Some(event) = event && let Err(reason) = emit_incoming(incoming, &event_slots, event) { return reason; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_connection<S, E>(
|
||||
writer: Arc<middleware::Writer<S>>,
|
||||
mut commands: mpsc::Receiver<ConnectionCommand>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
S: Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let mut heartbeat = tokio::time::interval_at(
|
||||
context.opened_at + DOWNSTREAM_PING_INTERVAL,
|
||||
DOWNSTREAM_PING_INTERVAL,
|
||||
);
|
||||
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut ping_sequence = 0_u64;
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => return PumpExitReason::LifecycleShutdown,
|
||||
command = commands.recv() => {
|
||||
let Some(command) = command else { return PumpExitReason::CoordinatorDropped; };
|
||||
let ConnectionCommand { message, context: write_context, acknowledged } = command;
|
||||
let closing = matches!(message, Message::Close(_));
|
||||
let result = match middleware::transform(message, contract::Direction::Outgoing, writer.clone(), context).await {
|
||||
Ok(Some(message)) => writer.write(message).await.map(|()| WriteOutcome::Written),
|
||||
Ok(None) => Ok(WriteOutcome::Suppressed),
|
||||
Err(error) => Err(ConnectionWriteError::Transport { message: error.to_string() }),
|
||||
};
|
||||
let reason = result.as_ref().err().map(|error| write_exit_reason(error, context));
|
||||
if let Err(error) = &result {
|
||||
tracing::debug!(websocket_connection_id = %context.connection_id, request_id = write_context.request_id.as_deref().unwrap_or(""), frame_phase = write_context.phase.as_str(), error = %error, "Responses WebSocket pump transport write failed");
|
||||
}
|
||||
let _ = acknowledged.send(result);
|
||||
if let Some(reason) = reason { return reason; }
|
||||
if closing { return PumpExitReason::ServerClose; }
|
||||
continue;
|
||||
}
|
||||
_ = heartbeat.tick() => {
|
||||
ping_sequence = ping_sequence.wrapping_add(1);
|
||||
Message::Ping(ping_sequence.to_be_bytes().to_vec().into())
|
||||
}
|
||||
};
|
||||
let message = match middleware::transform(
|
||||
message,
|
||||
contract::Direction::Outgoing,
|
||||
writer.clone(),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(message)) => message,
|
||||
Ok(None) => continue,
|
||||
Err(_) => return PumpExitReason::MiddlewareFailed,
|
||||
};
|
||||
if let Err(error) = writer.write(message).await {
|
||||
tracing::info!(websocket_connection_id = %context.connection_id, error = %error, "Responses WebSocket control write failed");
|
||||
return write_exit_reason(&error, context);
|
||||
}
|
||||
context
|
||||
.stats
|
||||
.ping_written_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_message<S, E>(
|
||||
socket: &mut S,
|
||||
message: Message,
|
||||
context: &PumpContext,
|
||||
) -> Result<(), ConnectionWriteError>
|
||||
where
|
||||
S: Sink<Message, Error = E> + Unpin,
|
||||
E: fmt::Display,
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => Err(ConnectionWriteError::Closed),
|
||||
result = timeout(context.config.write_timeout, socket.send(message)) => {
|
||||
match result {
|
||||
Ok(Ok(())) => { context.stats.record_write(context.opened_at); Ok(()) }
|
||||
Ok(Err(error)) => Err(ConnectionWriteError::Transport { message: error.to_string() }),
|
||||
Err(_) => Err(ConnectionWriteError::Timeout { timeout: context.config.write_timeout }),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_exit_reason(error: &ConnectionWriteError, context: &PumpContext) -> PumpExitReason {
|
||||
match error {
|
||||
ConnectionWriteError::Timeout { .. } => PumpExitReason::WriteTimeout,
|
||||
ConnectionWriteError::Transport { .. } => PumpExitReason::WriteError,
|
||||
ConnectionWriteError::Closed if context.cancellation.is_cancelled() => {
|
||||
PumpExitReason::LifecycleShutdown
|
||||
}
|
||||
ConnectionWriteError::Closed => PumpExitReason::PumpStopped,
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_incoming(
|
||||
incoming: &mpsc::Sender<PendingConnectionEvent>,
|
||||
slots: &Arc<Semaphore>,
|
||||
event: ConnectionEvent,
|
||||
) -> Result<(), PumpExitReason> {
|
||||
let permit = Arc::clone(slots)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| PumpExitReason::InboundOverload)?;
|
||||
incoming
|
||||
.try_send(PendingConnectionEvent {
|
||||
event,
|
||||
_permit: Some(permit),
|
||||
})
|
||||
.map_err(|error| match error {
|
||||
mpsc::error::TrySendError::Full(_) => PumpExitReason::InboundOverload,
|
||||
mpsc::error::TrySendError::Closed(_) => PumpExitReason::CoordinatorDropped,
|
||||
})
|
||||
}
|
||||
pub use handle::{ResponsesWebSocketConnection, spawn_connection};
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
//! 单 owner 驱动连接读写、心跳与有界入站队列
|
||||
|
||||
use super::{
|
||||
frame::*,
|
||||
middleware,
|
||||
state::{ConnectionCommand, ConnectionStats, PumpContext},
|
||||
};
|
||||
use axum::extract::ws::Message;
|
||||
use futures::{Sink, Stream, StreamExt};
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareHeader, websocket as contract},
|
||||
lifecycle::CancellationToken,
|
||||
};
|
||||
use std::{
|
||||
fmt,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
use tokio::{
|
||||
sync::{Semaphore, mpsc, oneshot},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_pump<S, E>(
|
||||
socket: S,
|
||||
commands: mpsc::Receiver<ConnectionCommand>,
|
||||
incoming: mpsc::Sender<PendingConnectionEvent>,
|
||||
exited: oneshot::Sender<PumpExitReason>,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
opened_at: Instant,
|
||||
expired: Arc<AtomicBool>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
config: ConnectionConfig,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) where
|
||||
S: Stream<Item = Result<Message, E>> + Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let context = Arc::new(PumpContext {
|
||||
connection_id,
|
||||
cancellation: cancellation.child_token(),
|
||||
opened_at,
|
||||
expired,
|
||||
stats,
|
||||
config,
|
||||
middleware,
|
||||
headers,
|
||||
});
|
||||
let _lifetime = middleware::CancelOnDrop(Some(context.cancellation.clone()));
|
||||
let (writer, mut reader) = socket.split();
|
||||
let writer = Arc::new(middleware::Writer::new(writer, context.clone()));
|
||||
let sender: Arc<dyn contract::Sender> = writer.clone();
|
||||
// 同一个受管任务同时驱动两侧;等待写入不能阻塞入站控制、关闭或取消
|
||||
let reason = tokio::select! {
|
||||
biased;
|
||||
reason = read_connection(&mut reader, &incoming, sender.clone(), &context) => reason,
|
||||
reason = write_connection(writer.clone(), commands, &context) => reason,
|
||||
};
|
||||
if matches!(reason, PumpExitReason::ClientClose) {
|
||||
if let Err(error) = writer.finish_peer_close(&cancellation).await {
|
||||
tracing::debug!(websocket_connection_id = %context.connection_id, %error, "Responses WebSocket close acknowledgement could not be flushed");
|
||||
}
|
||||
} else {
|
||||
context.cancellation.cancel();
|
||||
}
|
||||
let _ = exited.send(reason);
|
||||
tracing::debug!(websocket_connection_id = %context.connection_id, pump_exit_reason = reason.as_str(), "Responses WebSocket pump exited");
|
||||
}
|
||||
|
||||
async fn read_connection<S, E>(
|
||||
socket: &mut S,
|
||||
incoming: &mpsc::Sender<PendingConnectionEvent>,
|
||||
sender: Arc<dyn contract::Sender>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
S: Stream<Item = Result<Message, E>> + Unpin,
|
||||
E: fmt::Display,
|
||||
{
|
||||
let event_slots = Arc::new(Semaphore::new(INBOUND_EVENT_BUFFER));
|
||||
let deadline = tokio::time::sleep_until(context.opened_at + context.config.max_age);
|
||||
tokio::pin!(deadline);
|
||||
let mut deadline_elapsed = false;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => return PumpExitReason::LifecycleShutdown,
|
||||
() = &mut deadline, if !deadline_elapsed => {
|
||||
deadline_elapsed = true;
|
||||
context.expired.store(true, Ordering::Release);
|
||||
// 到期只阻止下一轮,当前响应仍由协调层按原有合同收尾
|
||||
match emit_incoming(incoming, &event_slots, ConnectionEvent::Expired) {
|
||||
Ok(()) | Err(PumpExitReason::InboundOverload) => {}
|
||||
Err(reason) => return reason,
|
||||
}
|
||||
}
|
||||
message = socket.next() => {
|
||||
if matches!(&message, Some(Ok(_))) { context.stats.record_read(context.opened_at); }
|
||||
let message = match message {
|
||||
Some(Ok(message)) => match middleware::transform(message, contract::Direction::Incoming, sender.clone(), context).await {
|
||||
Ok(Some(message)) => Some(Ok(message)),
|
||||
Ok(None) => continue,
|
||||
Err(_) => return PumpExitReason::MiddlewareFailed,
|
||||
},
|
||||
other => other,
|
||||
};
|
||||
let event = match message {
|
||||
Some(Ok(Message::Text(payload))) => Some(ConnectionEvent::Text(payload.to_string())),
|
||||
Some(Ok(Message::Binary(_))) => Some(ConnectionEvent::Binary),
|
||||
Some(Ok(Message::Ping(_))) => {
|
||||
// Axum/tungstenite 在继续读取时自动刷新 Pong,不能再手工发送一份
|
||||
context.stats.ping_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
Some(Ok(Message::Pong(_))) => {
|
||||
context.stats.pong_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
Some(Ok(Message::Close(_))) => return PumpExitReason::ClientClose,
|
||||
Some(Err(error)) => {
|
||||
tracing::info!(websocket_connection_id = %context.connection_id, error = %error, "Responses WebSocket receive failed");
|
||||
return PumpExitReason::ReadError;
|
||||
}
|
||||
None => return PumpExitReason::PeerEof,
|
||||
};
|
||||
if let Some(event) = event && let Err(reason) = emit_incoming(incoming, &event_slots, event) { return reason; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_connection<S, E>(
|
||||
writer: Arc<middleware::Writer<S>>,
|
||||
mut commands: mpsc::Receiver<ConnectionCommand>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
S: Sink<Message, Error = E> + Unpin + Send + 'static,
|
||||
E: fmt::Display + Send + 'static,
|
||||
{
|
||||
let mut heartbeat = tokio::time::interval_at(
|
||||
context.opened_at + DOWNSTREAM_PING_INTERVAL,
|
||||
DOWNSTREAM_PING_INTERVAL,
|
||||
);
|
||||
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut ping_sequence = 0_u64;
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
() = context.cancellation.cancelled() => return PumpExitReason::LifecycleShutdown,
|
||||
command = commands.recv() => {
|
||||
let Some(command) = command else { return PumpExitReason::CoordinatorDropped; };
|
||||
let ConnectionCommand { message, context: write_context, acknowledged } = command;
|
||||
let closing = matches!(message, Message::Close(_));
|
||||
let result = match middleware::transform(message, contract::Direction::Outgoing, writer.clone(), context).await {
|
||||
Ok(Some(message)) => writer.write(message).await.map(|()| WriteOutcome::Written),
|
||||
Ok(None) => Ok(WriteOutcome::Suppressed),
|
||||
Err(error) => Err(ConnectionWriteError::Transport { message: error.to_string() }),
|
||||
};
|
||||
let reason = result.as_ref().err().map(|error| write_exit_reason(error, context));
|
||||
if let Err(error) = &result {
|
||||
tracing::debug!(websocket_connection_id = %context.connection_id, request_id = write_context.request_id.as_deref().unwrap_or(""), frame_phase = write_context.phase.as_str(), error = %error, "Responses WebSocket pump transport write failed");
|
||||
}
|
||||
let _ = acknowledged.send(result);
|
||||
if let Some(reason) = reason { return reason; }
|
||||
if closing { return PumpExitReason::ServerClose; }
|
||||
continue;
|
||||
}
|
||||
_ = heartbeat.tick() => {
|
||||
ping_sequence = ping_sequence.wrapping_add(1);
|
||||
Message::Ping(ping_sequence.to_be_bytes().to_vec().into())
|
||||
}
|
||||
};
|
||||
let message = match middleware::transform(
|
||||
message,
|
||||
contract::Direction::Outgoing,
|
||||
writer.clone(),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(message)) => message,
|
||||
Ok(None) => continue,
|
||||
Err(_) => return PumpExitReason::MiddlewareFailed,
|
||||
};
|
||||
if let Err(error) = writer.write(message).await {
|
||||
tracing::info!(websocket_connection_id = %context.connection_id, error = %error, "Responses WebSocket control write failed");
|
||||
return write_exit_reason(&error, context);
|
||||
}
|
||||
context
|
||||
.stats
|
||||
.ping_written_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_exit_reason(error: &ConnectionWriteError, context: &PumpContext) -> PumpExitReason {
|
||||
match error {
|
||||
ConnectionWriteError::Timeout { .. } => PumpExitReason::WriteTimeout,
|
||||
ConnectionWriteError::Transport { .. } => PumpExitReason::WriteError,
|
||||
ConnectionWriteError::Closed if context.cancellation.is_cancelled() => {
|
||||
PumpExitReason::LifecycleShutdown
|
||||
}
|
||||
ConnectionWriteError::Closed => PumpExitReason::PumpStopped,
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_incoming(
|
||||
incoming: &mpsc::Sender<PendingConnectionEvent>,
|
||||
slots: &Arc<Semaphore>,
|
||||
event: ConnectionEvent,
|
||||
) -> Result<(), PumpExitReason> {
|
||||
let permit = Arc::clone(slots)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| PumpExitReason::InboundOverload)?;
|
||||
incoming
|
||||
.try_send(PendingConnectionEvent {
|
||||
event,
|
||||
_permit: Some(permit),
|
||||
})
|
||||
.map_err(|error| match error {
|
||||
mpsc::error::TrySendError::Full(_) => PumpExitReason::InboundOverload,
|
||||
mpsc::error::TrySendError::Closed(_) => PumpExitReason::CoordinatorDropped,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! 单连接共享的命令、运行预算与观测状态
|
||||
|
||||
use super::frame::{
|
||||
ConnectionConfig, ConnectionWriteError, OUTBOUND_COMMAND_BUFFER, WriteContext, WriteOutcome,
|
||||
};
|
||||
use axum::extract::ws::Message;
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareHeader},
|
||||
lifecycle::CancellationToken,
|
||||
};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
|
||||
};
|
||||
use tokio::{
|
||||
sync::{mpsc, oneshot},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
pub(super) struct ConnectionCommand {
|
||||
pub(super) message: Message,
|
||||
pub(super) context: WriteContext,
|
||||
pub(super) acknowledged: oneshot::Sender<Result<WriteOutcome, ConnectionWriteError>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ConnectionStats {
|
||||
pub(super) command_queue_high_water: AtomicUsize,
|
||||
pub(super) command_backpressure_count: AtomicU64,
|
||||
pub(super) ping_received_count: AtomicU64,
|
||||
pub(super) ping_written_count: AtomicU64,
|
||||
pub(super) pong_received_count: AtomicU64,
|
||||
pub(super) last_read_ms: AtomicU64,
|
||||
pub(super) last_write_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl ConnectionStats {
|
||||
pub(super) fn record_read(&self, opened_at: Instant) {
|
||||
self.last_read_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn record_write(&self, opened_at: Instant) {
|
||||
self.last_write_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn observe_command_queue(&self, sender: &mpsc::Sender<ConnectionCommand>) {
|
||||
let queued = OUTBOUND_COMMAND_BUFFER
|
||||
.saturating_sub(sender.capacity())
|
||||
.saturating_add(1)
|
||||
.min(OUTBOUND_COMMAND_BUFFER);
|
||||
self.command_queue_high_water
|
||||
.fetch_max(queued, Ordering::Relaxed);
|
||||
if sender.capacity() == 0 {
|
||||
self.command_backpressure_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PumpContext {
|
||||
pub(super) middleware: Option<FrozenMiddlewarePlan>,
|
||||
pub(super) headers: Arc<[MiddlewareHeader]>,
|
||||
pub(super) connection_id: Arc<str>,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) opened_at: Instant,
|
||||
pub(super) expired: Arc<AtomicBool>,
|
||||
pub(super) stats: Arc<ConnectionStats>,
|
||||
pub(super) config: ConnectionConfig,
|
||||
}
|
||||
@@ -5,24 +5,31 @@ use axum::{
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService,
|
||||
PreparedRootExecution, StartExecution, StartProviderExecution, StartedExecution,
|
||||
},
|
||||
middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareBody, MiddlewareContext, MiddlewareError,
|
||||
MiddlewareFrame, MiddlewareFraming, MiddlewareHeader, MiddlewareNext, MiddlewarePlan,
|
||||
MiddlewareRequest, MiddlewareResponse,
|
||||
},
|
||||
},
|
||||
error::GatewayError,
|
||||
lifecycle::CancellationToken,
|
||||
policy::ClientApiKeyId,
|
||||
routing::PublicModelId,
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::engine::execution::AuthenticatedClient;
|
||||
use gateway_core::engine::execution::ClientAuthenticationError;
|
||||
use gateway_core::engine::execution::ExecutionService;
|
||||
use gateway_core::engine::execution::PreparedRootExecution;
|
||||
use gateway_core::engine::execution::StartExecution;
|
||||
use gateway_core::engine::execution::StartProviderExecution;
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareBody;
|
||||
use gateway_core::engine::middleware::MiddlewareContext;
|
||||
use gateway_core::engine::middleware::MiddlewareError;
|
||||
use gateway_core::engine::middleware::MiddlewareFrame;
|
||||
use gateway_core::engine::middleware::MiddlewareFraming;
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_core::engine::middleware::MiddlewareNext;
|
||||
use gateway_core::engine::middleware::MiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareRequest;
|
||||
use gateway_core::engine::middleware::MiddlewareResponse;
|
||||
use gateway_core::error::GatewayError;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::routing::extensions::ExtensionSetId;
|
||||
use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
@@ -368,10 +375,11 @@ pub(super) fn resource_fixture(
|
||||
}
|
||||
|
||||
async fn published_fixture() -> AdminTestFixture {
|
||||
use gateway_core::{
|
||||
routing::{ConfigRevision, RuntimeSnapshot},
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::routing::ConfigRevision;
|
||||
use gateway_core::routing::RuntimeSnapshot;
|
||||
use gateway_core::routing::extensions::ExtensionSetId;
|
||||
use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
use std::sync::Arc;
|
||||
struct Lease;
|
||||
impl ExtensionSetLease for Lease {
|
||||
|
||||
@@ -546,7 +546,7 @@ pub(super) struct TestPluginPorts {
|
||||
impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPorts {
|
||||
async fn validate_target(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
) -> Result<(), AdminError> {
|
||||
Ok(())
|
||||
@@ -554,7 +554,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
|
||||
async fn start_callback(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: gateway_admin::model::plugins::management::StartPluginManagementCallback,
|
||||
_: &gateway_admin::model::auth::AdminRequestContext,
|
||||
@@ -564,7 +564,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn callback(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
_: &str,
|
||||
request: gateway_admin::model::plugins::management::PluginManagementRequest,
|
||||
@@ -576,14 +576,14 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn views(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
) -> Result<Vec<gateway_admin::model::plugins::management::PluginManagementView>, AdminError>
|
||||
{
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn resource(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
path: &str,
|
||||
public: bool,
|
||||
@@ -596,7 +596,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
}
|
||||
async fn handle(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::management::PluginManagementTarget,
|
||||
request: gateway_admin::model::plugins::management::PluginManagementRequest,
|
||||
) -> Result<gateway_admin::model::plugins::management::PluginManagementResponse, AdminError>
|
||||
@@ -611,7 +611,7 @@ impl gateway_admin::ports::plugin_management::PluginManagement for TestPluginPor
|
||||
impl gateway_admin::ports::plugins::PluginStateLifecycle for TestPluginPorts {
|
||||
async fn activate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: &gateway_admin::model::plugins::instances::PluginInstance,
|
||||
) -> Result<(), AdminError> {
|
||||
Ok(())
|
||||
@@ -621,7 +621,7 @@ impl gateway_admin::ports::plugins::PluginStateLifecycle for TestPluginPorts {
|
||||
}
|
||||
async fn migrate_state(
|
||||
&self,
|
||||
_: &gateway_core::runtime::extensions::ExtensionSetReference,
|
||||
_: &gateway_core::routing::extensions::ExtensionSetReference,
|
||||
_: gateway_admin::model::plugins::state::PluginStateTransition,
|
||||
) -> Result<(), AdminError> {
|
||||
panic!("unexpected state migration")
|
||||
@@ -647,7 +647,7 @@ impl gateway_admin::ports::plugins::PluginPreparation for TestPluginPorts {
|
||||
async fn prepare(
|
||||
&self,
|
||||
_: gateway_admin::model::plugins::instances::PluginInstanceSnapshot,
|
||||
) -> Result<gateway_core::runtime::extensions::ExtensionSetReference, AdminError> {
|
||||
) -> Result<gateway_core::routing::extensions::ExtensionSetReference, AdminError> {
|
||||
Err(AdminError::invalid("unused plugin fixture"))
|
||||
}
|
||||
}
|
||||
@@ -658,7 +658,7 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for TestPluginPorts
|
||||
&self,
|
||||
snapshot: &gateway_admin::model::plugins::instances::PluginInstanceSnapshot,
|
||||
_: Option<u64>,
|
||||
_: Option<&gateway_core::runtime::extensions::ExtensionSetReference>,
|
||||
_: Option<&gateway_core::routing::extensions::ExtensionSetReference>,
|
||||
) -> Option<BTreeMap<String, gateway_admin::model::plugins::instances::PluginInstanceRuntime>>
|
||||
{
|
||||
Some(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 检查 API 层依赖、源码与测试模块边界,以及管理路由约定
|
||||
//! 检查 API 层依赖与管理路由约定,模块布局和测试镜像由 workspace 架构检查覆盖
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
@@ -29,150 +29,6 @@ fn workspace_should_include_gateway_api() {
|
||||
assert!(workspace.contains("\"crates/gateway-api\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_tree_should_match_frozen_machine_manifest() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut actual = rust_files(&root.join("src"));
|
||||
actual.sort();
|
||||
let mut expected = vec![
|
||||
"src/admin/account_groups.rs",
|
||||
"src/admin/accounts/credentials.rs",
|
||||
"src/admin/accounts/handlers.rs",
|
||||
"src/admin/accounts/import_tasks.rs",
|
||||
"src/admin/accounts/mod.rs",
|
||||
"src/admin/accounts/presenter.rs",
|
||||
"src/admin/accounts/wire.rs",
|
||||
"src/admin/auth.rs",
|
||||
"src/admin/backups.rs",
|
||||
"src/admin/client_keys.rs",
|
||||
"src/admin/diagnostics.rs",
|
||||
"src/admin/extract.rs",
|
||||
"src/admin/mod.rs",
|
||||
"src/admin/observability/mod.rs",
|
||||
"src/admin/observability/presenter.rs",
|
||||
"src/admin/observability/query.rs",
|
||||
"src/admin/observability/routes.rs",
|
||||
"src/admin/observability/wire.rs",
|
||||
"src/admin/plugins/artifacts.rs",
|
||||
"src/admin/plugins/distribution.rs",
|
||||
"src/admin/plugins/instances.rs",
|
||||
"src/admin/plugins/management.rs",
|
||||
"src/admin/plugins/mod.rs",
|
||||
"src/admin/presenter.rs",
|
||||
"src/admin/proxies.rs",
|
||||
"src/admin/settings.rs",
|
||||
"src/admin/system.rs",
|
||||
"src/admin/wire.rs",
|
||||
"src/auth.rs",
|
||||
"src/health.rs",
|
||||
"src/key_usage/mod.rs",
|
||||
"src/key_usage/query.rs",
|
||||
"src/key_usage/presenter.rs",
|
||||
"src/lib.rs",
|
||||
"src/middleware/headers.rs",
|
||||
"src/middleware/mod.rs",
|
||||
"src/middleware/websocket.rs",
|
||||
"src/openai/middleware/http.rs",
|
||||
"src/openai/middleware/mod.rs",
|
||||
"src/openai/middleware/request.rs",
|
||||
"src/openai/middleware/session.rs",
|
||||
"src/openai/auth.rs",
|
||||
"src/openai/error.rs",
|
||||
"src/openai/images/http.rs",
|
||||
"src/openai/images/mod.rs",
|
||||
"src/openai/live/http.rs",
|
||||
"src/openai/live/mod.rs",
|
||||
"src/openai/live/websocket.rs",
|
||||
"src/openai/mod.rs",
|
||||
"src/openai/models.rs",
|
||||
"src/openai/endpoint.rs",
|
||||
"src/openai/responses/error.rs",
|
||||
"src/openai/responses/http.rs",
|
||||
"src/openai/responses/mod.rs",
|
||||
"src/openai/responses/request.rs",
|
||||
"src/openai/responses/response.rs",
|
||||
"src/openai/responses/validation.rs",
|
||||
"src/openai/responses/websocket/connection/mod.rs",
|
||||
"src/openai/responses/websocket/connection/middleware.rs",
|
||||
"src/openai/responses/websocket/forward.rs",
|
||||
"src/openai/responses/websocket/mod.rs",
|
||||
"src/openai/responses/websocket/protocol.rs",
|
||||
"src/openai/router.rs",
|
||||
"src/openai/search/http.rs",
|
||||
"src/openai/search/mod.rs",
|
||||
"src/openai/service.rs",
|
||||
"src/openai/usage.rs",
|
||||
"src/provider.rs",
|
||||
"src/session_cookie.rs",
|
||||
"src/time.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tree_should_match_frozen_rust_mirror() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let mut actual = rust_files(&root.join("tests"));
|
||||
actual.sort();
|
||||
let mut expected = vec![
|
||||
"tests/admin/account_groups.rs",
|
||||
"tests/admin/accounts/handlers.rs",
|
||||
"tests/admin/accounts/import_tasks.rs",
|
||||
"tests/admin/accounts/mod.rs",
|
||||
"tests/admin/accounts/presenter.rs",
|
||||
"tests/admin/auth.rs",
|
||||
"tests/admin/client_keys.rs",
|
||||
"tests/admin/errors.rs",
|
||||
"tests/admin/error_diagnostics.rs",
|
||||
"tests/admin/mod.rs",
|
||||
"tests/admin/observability/mod.rs",
|
||||
"tests/admin/observability/query.rs",
|
||||
"tests/admin/observability/response.rs",
|
||||
"tests/admin/plugins/management.rs",
|
||||
"tests/admin/plugins/mod.rs",
|
||||
"tests/admin/proxies.rs",
|
||||
"tests/admin/settings.rs",
|
||||
"tests/admin/system.rs",
|
||||
"tests/admin/wire.rs",
|
||||
"tests/architecture.rs",
|
||||
"tests/auth.rs",
|
||||
"tests/support.rs",
|
||||
"tests/health.rs",
|
||||
"tests/key_usage/mod.rs",
|
||||
"tests/key_usage/fixtures.rs",
|
||||
"tests/main.rs",
|
||||
"tests/middleware/mod.rs",
|
||||
"tests/middleware/websocket.rs",
|
||||
"tests/openai/middleware/mod.rs",
|
||||
"tests/openai/auth.rs",
|
||||
"tests/openai/error.rs",
|
||||
"tests/openai/images/mod.rs",
|
||||
"tests/openai/mod.rs",
|
||||
"tests/openai/models.rs",
|
||||
"tests/openai/endpoint.rs",
|
||||
"tests/openai/responses/http.rs",
|
||||
"tests/openai/responses/mod.rs",
|
||||
"tests/openai/responses/request.rs",
|
||||
"tests/openai/responses/websocket/connection/mod.rs",
|
||||
"tests/openai/responses/websocket/connection/middleware.rs",
|
||||
"tests/openai/responses/websocket/forward.rs",
|
||||
"tests/openai/responses/websocket/interrupt.rs",
|
||||
"tests/openai/responses/websocket/mod.rs",
|
||||
"tests/openai/responses/websocket/protocol.rs",
|
||||
"tests/openai/live/mod.rs",
|
||||
"tests/openai/live/websocket.rs",
|
||||
"tests/openai/router.rs",
|
||||
"tests/openai/search/mod.rs",
|
||||
"tests/openai/usage.rs",
|
||||
"tests/provider.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_routes_should_use_only_get_post_and_static_paths() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/admin");
|
||||
@@ -192,20 +48,6 @@ fn admin_routes_should_use_only_get_post_and_static_paths() {
|
||||
);
|
||||
}
|
||||
|
||||
fn rust_files(root: &Path) -> Vec<String> {
|
||||
let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
all_files(root)
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().and_then(|value| value.to_str()) == Some("rs"))
|
||||
.map(|path| {
|
||||
path.strip_prefix(manifest)
|
||||
.expect("path below manifest")
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn all_files(root: &Path) -> Vec<std::path::PathBuf> {
|
||||
let mut files = Vec::new();
|
||||
let mut pending = vec![root.to_path_buf()];
|
||||
|
||||
@@ -16,15 +16,18 @@ use axum::{
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{future::BoxFuture, stream};
|
||||
use gateway_core::{
|
||||
engine::middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareContext, MiddlewareError, MiddlewareNext, MiddlewarePlan,
|
||||
MiddlewareRequest, MiddlewareResponse,
|
||||
},
|
||||
lifecycle::CancellationToken,
|
||||
middleware::http as contract,
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareContext;
|
||||
use gateway_core::engine::middleware::MiddlewareError;
|
||||
use gateway_core::engine::middleware::MiddlewareNext;
|
||||
use gateway_core::engine::middleware::MiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareRequest;
|
||||
use gateway_core::engine::middleware::MiddlewareResponse;
|
||||
use gateway_core::engine::middleware::http as contract;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::routing::extensions::ExtensionSetId;
|
||||
use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
use http_body::Frame;
|
||||
use http_body_util::{BodyExt as _, StreamBody};
|
||||
use tower::ServiceExt as _;
|
||||
@@ -70,12 +73,12 @@ impl MiddlewarePlan for Plan {
|
||||
}
|
||||
fn handle_websocket(
|
||||
&self,
|
||||
context: gateway_core::middleware::websocket::Context,
|
||||
message: gateway_core::middleware::websocket::Message,
|
||||
next: gateway_core::middleware::websocket::Next,
|
||||
context: gateway_core::engine::middleware::websocket::Context,
|
||||
message: gateway_core::engine::middleware::websocket::Message,
|
||||
next: gateway_core::engine::middleware::websocket::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<Option<gateway_core::middleware::websocket::Message>, MiddlewareError>,
|
||||
Result<Option<gateway_core::engine::middleware::websocket::Message>, MiddlewareError>,
|
||||
> {
|
||||
assert!(context.plan.is_some());
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -103,11 +106,13 @@ impl MiddlewarePlan for Plan {
|
||||
}
|
||||
fn handle_service(
|
||||
&self,
|
||||
context: gateway_core::middleware::service::Context,
|
||||
context: gateway_core::engine::middleware::service::Context,
|
||||
input: serde_json::Value,
|
||||
next: gateway_core::middleware::service::Next,
|
||||
) -> BoxFuture<'static, Result<serde_json::Value, gateway_core::middleware::service::Error>>
|
||||
{
|
||||
next: gateway_core::engine::middleware::service::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<serde_json::Value, gateway_core::engine::middleware::service::Error>,
|
||||
> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
assert_eq!(context.operation, "settings.load");
|
||||
assert_eq!(context.request_id, "service-request");
|
||||
@@ -146,7 +151,7 @@ impl MiddlewarePlan for Plan {
|
||||
response.extensions_mut().insert(contract::upgrade::Upgraded::new(status, Box::pin(async move {
|
||||
let session = session.await.map_err(|_| MiddlewareError::Fault)?;
|
||||
if idle { return std::future::pending().await; }
|
||||
use gateway_core::middleware::websocket::{Kind, Message};
|
||||
use gateway_core::engine::middleware::websocket::{Kind, Message};
|
||||
let receive = session.receive();
|
||||
tokio::pin!(receive);
|
||||
tokio::select! {
|
||||
|
||||
@@ -374,9 +374,9 @@ fn chatgpt_remote_suffix_created_by_header_truncation_should_not_skip_the_gate()
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission() {
|
||||
use gateway_core::{
|
||||
engine::middleware::*, middleware::http as contract, runtime::extensions::*,
|
||||
};
|
||||
use gateway_core::engine::middleware::http as contract;
|
||||
use gateway_core::engine::middleware::*;
|
||||
use gateway_core::routing::extensions::*;
|
||||
struct Lease;
|
||||
impl ExtensionSetLease for Lease {
|
||||
fn is_ready(&self) -> bool {
|
||||
|
||||
@@ -10,15 +10,22 @@ use std::{
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareBody, MiddlewareContext, MiddlewareError, MiddlewareFrame,
|
||||
MiddlewareFraming, MiddlewareHeader, MiddlewareMount, MiddlewareNext, MiddlewarePlan,
|
||||
MiddlewareRequest, MiddlewareResponse,
|
||||
},
|
||||
operation::OperationKind,
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareBody;
|
||||
use gateway_core::engine::middleware::MiddlewareContext;
|
||||
use gateway_core::engine::middleware::MiddlewareError;
|
||||
use gateway_core::engine::middleware::MiddlewareFrame;
|
||||
use gateway_core::engine::middleware::MiddlewareFraming;
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_core::engine::middleware::MiddlewareMount;
|
||||
use gateway_core::engine::middleware::MiddlewareNext;
|
||||
use gateway_core::engine::middleware::MiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareRequest;
|
||||
use gateway_core::engine::middleware::MiddlewareResponse;
|
||||
use gateway_core::operation::OperationKind;
|
||||
use gateway_core::routing::extensions::ExtensionSetId;
|
||||
use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
|
||||
/// OpenAI API 入口共用的中间件夹具;授权和 RPC 由 Runtime 行为测试覆盖
|
||||
#[derive(Debug, Default)]
|
||||
@@ -30,7 +37,8 @@ pub(super) struct RequestMiddleware {
|
||||
pub(super) response_actions: Mutex<VecDeque<ResponseFrameAction>>,
|
||||
pub(super) response_headers: Vec<MiddlewareHeader>,
|
||||
pub(super) short_circuit_json: Option<Bytes>,
|
||||
pub(super) settings: Option<fn(&mut gateway_core::settings::ExecutionSettings)>,
|
||||
pub(super) settings:
|
||||
Option<fn(&mut gateway_core::routing::request_settings::ExecutionSettings)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
|
||||
+94
-8
@@ -2,14 +2,100 @@
|
||||
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use gateway_core::{
|
||||
engine::middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareContext, MiddlewareError, MiddlewareNext, MiddlewarePlan,
|
||||
MiddlewareRequest, MiddlewareResponse,
|
||||
},
|
||||
middleware::{http as http_contract, websocket as core},
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareContext;
|
||||
use gateway_core::engine::middleware::MiddlewareError;
|
||||
use gateway_core::engine::middleware::MiddlewareNext;
|
||||
use gateway_core::engine::middleware::MiddlewarePlan;
|
||||
use gateway_core::engine::middleware::MiddlewareRequest;
|
||||
use gateway_core::engine::middleware::MiddlewareResponse;
|
||||
use gateway_core::engine::middleware::http as http_contract;
|
||||
use gateway_core::engine::middleware::websocket as core;
|
||||
use gateway_core::routing::extensions::ExtensionSetId;
|
||||
use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn client_close_does_not_flush_a_retained_sender_with_an_inflight_write() {
|
||||
#[derive(Default)]
|
||||
struct RetainedSenderPlan(Mutex<Option<Arc<dyn core::Sender>>>);
|
||||
impl fmt::Debug for RetainedSenderPlan {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("RetainedSenderPlan")
|
||||
}
|
||||
}
|
||||
impl MiddlewarePlan for RetainedSenderPlan {
|
||||
fn has_websocket(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn handle_websocket(
|
||||
&self,
|
||||
context: core::Context,
|
||||
message: core::Message,
|
||||
next: core::Next,
|
||||
) -> BoxFuture<'static, Result<Option<core::Message>, MiddlewareError>> {
|
||||
*self.0.lock().unwrap() = Some(context.sender);
|
||||
next.run(message)
|
||||
}
|
||||
fn handle(
|
||||
&self,
|
||||
_: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<MiddlewareResponse, MiddlewareError>> {
|
||||
next.run(request)
|
||||
}
|
||||
}
|
||||
|
||||
let plan = Arc::new(RetainedSenderPlan::default());
|
||||
let frozen = FrozenMiddlewarePlan::new(
|
||||
plan.clone(),
|
||||
ExtensionSetReference::new(
|
||||
ExtensionSetId::new("retained-writer".into()).unwrap(),
|
||||
Arc::new(Lease),
|
||||
),
|
||||
);
|
||||
let (incoming, received) = unbounded_channel();
|
||||
let (written, mut output) = unbounded_channel();
|
||||
let flush_started = Arc::new(AtomicBool::new(false));
|
||||
let socket = TestSocket {
|
||||
incoming: received,
|
||||
written,
|
||||
stall_writes: true,
|
||||
flush_started: flush_started.clone(),
|
||||
dropped: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
let mut connection = spawn_connection(
|
||||
socket,
|
||||
Arc::from("retained-writer"),
|
||||
CancellationToken::new(),
|
||||
ConnectionConfig::PRODUCTION,
|
||||
Some(frozen),
|
||||
Arc::from([]),
|
||||
);
|
||||
incoming.send(Ok(Message::Text("capture".into()))).unwrap();
|
||||
assert!(matches!(
|
||||
connection.next_event().await,
|
||||
Some(ConnectionEvent::Text(_))
|
||||
));
|
||||
let sender = plan.0.lock().unwrap().clone().unwrap();
|
||||
let mut write = sender.send(core::Message {
|
||||
kind: core::Kind::Text,
|
||||
payload: Bytes::from_static(b"pending"),
|
||||
});
|
||||
assert!(futures::poll!(write.as_mut()).is_pending());
|
||||
// 写 future 在 pump 的 select 之外持锁,而且暂不继续 poll 以响应取消
|
||||
incoming.send(Ok(Message::Close(None))).unwrap();
|
||||
let reason = tokio::time::timeout(Duration::from_millis(50), connection.wait_for_exit())
|
||||
.await
|
||||
.expect("peer close must not wait for a retained sender");
|
||||
assert_eq!(reason, PumpExitReason::ClientClose);
|
||||
assert!(!flush_started.load(Ordering::Acquire));
|
||||
assert!(write.await.is_err());
|
||||
assert!(matches!(output.try_recv(), Err(TryRecvError::Empty)));
|
||||
plan.0.lock().unwrap().take();
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Plan {
|
||||
block_terminal: bool,
|
||||
|
||||
@@ -234,6 +234,7 @@ struct TestSocket {
|
||||
incoming: UnboundedReceiver<Result<Message, TestSocketError>>,
|
||||
written: UnboundedSender<Message>,
|
||||
stall_writes: bool,
|
||||
flush_started: Arc<AtomicBool>,
|
||||
dropped: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
@@ -268,7 +269,12 @@ impl Sink<Message> for TestSocket {
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
self.flush_started.store(true, Ordering::Release);
|
||||
if self.stall_writes {
|
||||
Poll::Pending
|
||||
} else {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
@@ -286,6 +292,7 @@ struct PumpHarness {
|
||||
connection: ResponsesWebSocketConnection,
|
||||
incoming: UnboundedSender<Result<Message, TestSocketError>>,
|
||||
written: UnboundedReceiver<Message>,
|
||||
flush_started: Arc<AtomicBool>,
|
||||
dropped: Arc<AtomicBool>,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
@@ -294,11 +301,13 @@ fn test_connection(stall_writes: bool) -> PumpHarness {
|
||||
let (incoming_tx, incoming_rx) = unbounded_channel();
|
||||
let (written_tx, written_rx) = unbounded_channel();
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let flush_started = Arc::new(AtomicBool::new(false));
|
||||
let cancellation = CancellationToken::new();
|
||||
let socket = TestSocket {
|
||||
incoming: incoming_rx,
|
||||
written: written_tx,
|
||||
stall_writes,
|
||||
flush_started: Arc::clone(&flush_started),
|
||||
dropped: Arc::clone(&dropped),
|
||||
};
|
||||
let connection = spawn_connection(
|
||||
@@ -313,6 +322,7 @@ fn test_connection(stall_writes: bool) -> PumpHarness {
|
||||
connection,
|
||||
incoming: incoming_tx,
|
||||
written: written_rx,
|
||||
flush_started,
|
||||
dropped,
|
||||
cancellation,
|
||||
}
|
||||
@@ -404,6 +414,62 @@ async fn client_close_should_take_priority_over_queued_business_frames() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn client_close_flush_has_a_short_deadline_without_sending_another_frame() {
|
||||
let PumpHarness {
|
||||
mut connection,
|
||||
incoming,
|
||||
mut written,
|
||||
flush_started,
|
||||
dropped,
|
||||
..
|
||||
} = test_connection(true);
|
||||
incoming.send(Ok(Message::Close(None))).expect("close peer");
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
let reason = tokio::time::timeout(Duration::from_secs(2), connection.wait_for_exit())
|
||||
.await
|
||||
.expect("peer close cannot consume the ordinary write timeout");
|
||||
assert_eq!(reason, PumpExitReason::ClientClose);
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(1));
|
||||
assert!(flush_started.load(Ordering::Acquire));
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
assert!(
|
||||
written.recv().await.is_none(),
|
||||
"do not send a duplicate Close"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn lifecycle_shutdown_interrupts_a_pending_client_close_flush() {
|
||||
let PumpHarness {
|
||||
mut connection,
|
||||
incoming,
|
||||
flush_started,
|
||||
cancellation,
|
||||
dropped,
|
||||
..
|
||||
} = test_connection(true);
|
||||
incoming.send(Ok(Message::Close(None))).expect("close peer");
|
||||
for _ in 0..100 {
|
||||
if flush_started.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(
|
||||
flush_started.load(Ordering::Acquire),
|
||||
"close flush must start"
|
||||
);
|
||||
cancellation.cancel();
|
||||
|
||||
let reason = tokio::time::timeout(Duration::from_millis(50), connection.wait_for_exit())
|
||||
.await
|
||||
.expect("host cancellation must not wait for the close timeout");
|
||||
assert_eq!(reason, PumpExitReason::ClientClose);
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_business_queue_should_not_block_lifecycle_shutdown() {
|
||||
let PumpHarness {
|
||||
@@ -988,6 +1054,40 @@ async fn requests_deferred_during_active_response_share_the_inbound_queue_limit(
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_normal_close_after_completed_response_receives_normal_close_reply() {
|
||||
let (trace, mut socket, server) = start_active_response().await;
|
||||
trace.release_terminal.notify_one();
|
||||
loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("terminal response timeout")
|
||||
.expect("response remains open")
|
||||
.expect("valid response frame");
|
||||
if let ClientMessage::Text(text) = message {
|
||||
let event: Value = serde_json::from_str(&text).expect("response JSON");
|
||||
if event["type"] == "response.completed" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
socket
|
||||
.close(Some(tokio_tungstenite::tungstenite::protocol::CloseFrame {
|
||||
code: CloseCode::Normal,
|
||||
reason: "client finished".into(),
|
||||
}))
|
||||
.await
|
||||
.expect("send normal close");
|
||||
// close() 只发送本端关闭帧;必须读取对端应答,才能证明握手正常完成
|
||||
let reply = tokio::time::timeout(Duration::from_secs(2), socket.next()).await;
|
||||
server.abort();
|
||||
assert!(
|
||||
matches!(&reply, Ok(Some(Ok(ClientMessage::Close(Some(frame)))))
|
||||
if frame.code == CloseCode::Normal && frame.reason == "client finished"),
|
||||
"client must receive the close acknowledgement, got {reply:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_close_should_cancel_active_execution_without_starting_queued_requests() {
|
||||
let (trace, mut socket, server) = start_active_response().await;
|
||||
@@ -1009,6 +1109,12 @@ async fn client_close_should_cancel_active_execution_without_starting_queued_req
|
||||
.expect("cancel active execution promptly");
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), 1);
|
||||
assert!(trace.cancelled.load(Ordering::Acquire));
|
||||
let reply = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("close reply timeout")
|
||||
.expect("close reply")
|
||||
.expect("clean close handshake");
|
||||
assert!(matches!(reply, ClientMessage::Close(None)));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
@@ -1073,6 +1179,7 @@ async fn stalled_heartbeat_can_be_cancelled_without_waiting_for_write_timeout()
|
||||
written: _written,
|
||||
dropped,
|
||||
cancellation,
|
||||
..
|
||||
} = test_connection(true);
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
@@ -19,7 +19,7 @@ use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::upstream::{OpaqueUpstreamValue, UpstreamSendState};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
@@ -10,7 +10,7 @@ use futures::future::BoxFuture;
|
||||
|
||||
use crate::{
|
||||
policy::{ClientApiKeyId, PlaintextClientApiKey},
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
};
|
||||
|
||||
pub const MAXIMUM_AUTHORIZATION_BYTES: usize = 8 * 1024;
|
||||
@@ -18,24 +18,29 @@ pub const MAXIMUM_AUTHORIZATION_BYTES: usize = 8 * 1024;
|
||||
/// API 从数据面 Authorization 头构造的有界认证信封
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ClientAuthenticationRequest {
|
||||
settings: Option<crate::settings::RequestSettings>,
|
||||
settings: Option<crate::routing::request_settings::RequestSettings>,
|
||||
authorization: Arc<str>,
|
||||
native_bearer: Option<PlaintextClientApiKey>,
|
||||
}
|
||||
|
||||
impl ClientAuthenticationRequest {
|
||||
#[must_use]
|
||||
pub fn with_settings(mut self, settings: crate::settings::RequestSettings) -> Self {
|
||||
pub fn with_settings(
|
||||
mut self,
|
||||
settings: crate::routing::request_settings::RequestSettings,
|
||||
) -> Self {
|
||||
self.settings = Some(settings);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn settings(&self) -> Option<&crate::settings::RequestSettings> {
|
||||
pub fn settings(&self) -> Option<&crate::routing::request_settings::RequestSettings> {
|
||||
self.settings.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn take_settings(&mut self) -> Option<crate::settings::RequestSettings> {
|
||||
pub(crate) fn take_settings(
|
||||
&mut self,
|
||||
) -> Option<crate::routing::request_settings::RequestSettings> {
|
||||
self.settings.take()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
//! 请求准入租约及费用结算的释放边界
|
||||
|
||||
use super::failure::record_resource_failure;
|
||||
use crate::{
|
||||
engine::{
|
||||
ModelRequestId,
|
||||
admission::ClientAdmissionPort,
|
||||
budget::{ClientBudgetCharge, ClientBudgetPort},
|
||||
},
|
||||
lifecycle::LeaseGuard,
|
||||
policy::ClientApiKeyId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) struct AdmissionLease {
|
||||
pub(super) renewal: Option<Box<dyn LeaseGuard>>,
|
||||
pub(super) armed: bool,
|
||||
pub(super) port: Arc<dyn ClientAdmissionPort>,
|
||||
pub(super) diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
pub(super) client_api_key_id: ClientApiKeyId,
|
||||
pub(super) model_request_id: ModelRequestId,
|
||||
}
|
||||
|
||||
pub(super) async fn settle_budget(
|
||||
port: &dyn ClientBudgetPort,
|
||||
charge: ClientBudgetCharge,
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
) {
|
||||
let request_id = charge.request_id.clone();
|
||||
if let Err(error) = port.settle(charge).await {
|
||||
record_resource_failure(
|
||||
diagnostics,
|
||||
&request_id,
|
||||
"settle_client_budget",
|
||||
"Client budget settlement failed; storage will retry on the next request",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionLease {
|
||||
pub(super) async fn release(mut self) {
|
||||
self.renewal.take();
|
||||
if let Err(error) = self
|
||||
.port
|
||||
.release(&self.client_api_key_id, &self.model_request_id)
|
||||
.await
|
||||
{
|
||||
record_resource_failure(
|
||||
self.diagnostics.as_ref(),
|
||||
&self.model_request_id,
|
||||
"release_client_admission",
|
||||
"Client admission release failed; lease TTL remains active",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdmissionLease {
|
||||
fn drop(&mut self) {
|
||||
self.renewal.take();
|
||||
if self.armed {
|
||||
self.port
|
||||
.abandon(&self.client_api_key_id, &self.model_request_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
//! 执行请求、认证事实与对外会话端口
|
||||
|
||||
use crate::{
|
||||
engine::{
|
||||
CoordinatedEvent, EngineError, ModelRequestId,
|
||||
authentication::ClientAuthenticationRequest,
|
||||
continuation::PreviousResponseId,
|
||||
extensions::ExtensionCallScope,
|
||||
middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareAuthority, MiddlewareContext, MiddlewareTarget,
|
||||
},
|
||||
nested::ExecutionEffects,
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
event::{ProviderEvent, ProviderResponseHeader},
|
||||
identity::ProviderKind,
|
||||
lifecycle::{CancellationToken, Deadline},
|
||||
operation::Operation,
|
||||
policy::{ClientApiKeyId, ClientPolicy},
|
||||
routing::{
|
||||
ProviderCatalogUnavailable, PublicModelDescriptor, PublicModelId, RuntimeSnapshot,
|
||||
UpstreamModelId, request_settings::RequestSettings,
|
||||
},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{
|
||||
fmt,
|
||||
net::IpAddr,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
HttpJson,
|
||||
HttpSse,
|
||||
WebSocket,
|
||||
InternalProbe,
|
||||
InternalPlugin,
|
||||
}
|
||||
|
||||
impl ClientTransport {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HttpJson => "http_json",
|
||||
Self::HttpSse => "http_sse",
|
||||
Self::WebSocket => "websocket",
|
||||
Self::InternalProbe => "internal",
|
||||
Self::InternalPlugin => "internal_plugin",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// API 解码后交给 Core 的稳定请求元数据
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ExecutionRequestMetadata {
|
||||
pub protocol: String,
|
||||
pub endpoint: String,
|
||||
pub transport: ClientTransport,
|
||||
pub stream: bool,
|
||||
pub client_ip: Option<IpAddr>,
|
||||
pub user_agent: Option<String>,
|
||||
pub previous_response_id: Option<PreviousResponseId>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AuthenticatedClient {
|
||||
pub(super) settings: Option<RequestSettings>,
|
||||
pub(super) snapshot: Arc<RuntimeSnapshot>,
|
||||
pub(super) policy: ClientPolicy,
|
||||
pub(super) authentication: Option<ClientAuthenticationRequest>,
|
||||
}
|
||||
|
||||
impl AuthenticatedClient {
|
||||
pub(super) fn execution_timeout(&self) -> Option<Duration> {
|
||||
self.settings
|
||||
.as_ref()
|
||||
.and_then(|settings| settings.execution_timeout(self.policy.key_id()))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn request_settings(&self) -> Option<&RequestSettings> {
|
||||
self.settings.as_ref()
|
||||
}
|
||||
|
||||
/// 后续仍须重新认证;这里只更新下一次认证使用的冻结配置
|
||||
#[must_use]
|
||||
pub fn with_request_settings(mut self, settings: RequestSettings) -> Self {
|
||||
self.settings = Some(settings);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn snapshot(&self) -> &Arc<RuntimeSnapshot> {
|
||||
&self.snapshot
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn policy(&self) -> &ClientPolicy {
|
||||
&self.policy
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AuthenticatedClient {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AuthenticatedClient")
|
||||
.field("key_id", &self.policy.key_id())
|
||||
.field("revision", &self.snapshot.revision())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ClientAuthenticationError {
|
||||
#[error("client API key is invalid")]
|
||||
InvalidKey,
|
||||
#[error("runtime snapshot is unavailable")]
|
||||
SnapshotUnavailable,
|
||||
#[error("frontend authentication provider is unavailable")]
|
||||
ProviderUnavailable,
|
||||
}
|
||||
|
||||
pub struct StartExecution {
|
||||
pub client: AuthenticatedClient,
|
||||
pub public_model: PublicModelId,
|
||||
pub operation: Operation,
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
/// 入口中间件开始前冻结的根请求身份与生命周期
|
||||
///
|
||||
/// 中间件短路时本值直接释放;调用 `next` 后必须原样交回
|
||||
/// [`ExecutionService::start_prepared`],不能重新认证为另一个 Key
|
||||
pub struct PreparedRootExecution {
|
||||
pub(super) extension_scope: ExtensionCallScope,
|
||||
pub(super) response_control: crate::engine::response_control::ResponseControl,
|
||||
pub(super) client: AuthenticatedClient,
|
||||
pub(super) request_id: ModelRequestId,
|
||||
pub(super) started_at: SystemTime,
|
||||
pub(super) deadline_at: Deadline,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) execution_effects: Arc<ExecutionEffects>,
|
||||
pub(super) execution_effects_baseline: usize,
|
||||
}
|
||||
|
||||
impl PreparedRootExecution {
|
||||
#[must_use]
|
||||
pub fn request_settings(&self) -> RequestSettings {
|
||||
self.client
|
||||
.settings
|
||||
.clone()
|
||||
.unwrap_or_else(|| RequestSettings::new(self.client.snapshot.clone()))
|
||||
.with_execution(
|
||||
&self.client.policy,
|
||||
self.deadline_at
|
||||
.at()
|
||||
.map(|at| duration_ms(at.duration_since(self.started_at).unwrap_or_default())),
|
||||
)
|
||||
}
|
||||
|
||||
/// 入口延续发起实例集合;保留已经交给调用方的取消句柄
|
||||
#[must_use]
|
||||
pub fn with_extension_scope(mut self, scope: ExtensionCallScope) -> Self {
|
||||
self.extension_scope = scope;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn extension_scope(&self) -> ExtensionCallScope {
|
||||
self.extension_scope.clone()
|
||||
}
|
||||
|
||||
/// 设置已经在改写时完成编译;此处只将同一份结果应用到冻结身份和生命周期
|
||||
pub fn apply_settings(&mut self, settings: &RequestSettings) -> Result<(), GatewayError> {
|
||||
let deadline = settings
|
||||
.execution_deadline(self.client.policy.key_id(), self.started_at)
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"execution settings are invalid",
|
||||
)
|
||||
})?;
|
||||
self.client.policy = settings.apply_policy(self.client.policy.clone());
|
||||
self.client.snapshot = settings.snapshot();
|
||||
self.client.settings = Some(settings.clone());
|
||||
self.deadline_at = deadline;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn response_control(&self) -> crate::engine::response_control::ResponseControl {
|
||||
self.response_control.clone()
|
||||
}
|
||||
|
||||
pub(super) fn new(client: AuthenticatedClient) -> Result<Self, GatewayError> {
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = Deadline::from_timeout(started_at, client.execution_timeout())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "system clock is invalid")
|
||||
})?;
|
||||
let execution_effects = Arc::new(ExecutionEffects::default());
|
||||
let execution_effects_baseline = execution_effects.epoch();
|
||||
Ok(Self {
|
||||
extension_scope: ExtensionCallScope::default(),
|
||||
client,
|
||||
response_control: crate::engine::response_control::ResponseControl::default(),
|
||||
request_id: new_request_id()?,
|
||||
started_at,
|
||||
deadline_at,
|
||||
cancellation: CancellationToken::new(),
|
||||
execution_effects,
|
||||
execution_effects_baseline,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> &ModelRequestId {
|
||||
&self.request_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn client(&self) -> &AuthenticatedClient {
|
||||
&self.client
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn started_at(&self) -> SystemTime {
|
||||
self.started_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn deadline_at(&self) -> Deadline {
|
||||
self.deadline_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn cancellation(&self) -> CancellationToken {
|
||||
self.cancellation.clone()
|
||||
}
|
||||
|
||||
/// 以冻结的入口身份补齐 Runtime-only binding 事实;调用位置事实由 API/Core 提供
|
||||
#[must_use]
|
||||
pub fn middleware_context(
|
||||
&self,
|
||||
mut target: MiddlewareTarget,
|
||||
extension_scope: ExtensionCallScope,
|
||||
) -> MiddlewareContext {
|
||||
target.request_id = self.request_id.clone();
|
||||
let account_group_ids = self
|
||||
.client
|
||||
.policy
|
||||
.account_scope()
|
||||
.routing_snapshot()
|
||||
.groups_snapshot()
|
||||
.iter()
|
||||
.map(|group| group.id().clone())
|
||||
.collect::<Vec<_>>();
|
||||
MiddlewareContext::new(
|
||||
target,
|
||||
MiddlewareAuthority {
|
||||
client_key_id: self.client.policy.key_id().clone(),
|
||||
account_group_ids: Arc::from(account_group_ids),
|
||||
cancellation: self.cancellation.clone(),
|
||||
deadline: self.deadline_at,
|
||||
extension_scope,
|
||||
execution_effects: Some(Arc::clone(&self.execution_effects)),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// 入口中间件调用 `next` 后交给 Core 的已解码执行输入
|
||||
pub struct PreparedExecutionRequest {
|
||||
pub public_model: PublicModelId,
|
||||
pub operation: Operation,
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
/// 启动一个由协议 adapter 明确绑定到 Provider 自有端点的请求
|
||||
pub struct StartProviderExecution {
|
||||
pub client: AuthenticatedClient,
|
||||
pub provider: ProviderKind,
|
||||
/// Provider 自有模型端点可固定上游模型;非模型端点保持 `None`
|
||||
pub upstream_model: Option<UpstreamModelId>,
|
||||
pub operation: Operation,
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
pub struct StartedExecution {
|
||||
pub request_id: ModelRequestId,
|
||||
pub created_at: SystemTime,
|
||||
pub stream: bool,
|
||||
pub session: Box<dyn ExecutionSession>,
|
||||
}
|
||||
|
||||
pub trait ExecutionSession: Send {
|
||||
fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
crate::diagnostics::TraceContext::default()
|
||||
}
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>>;
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>>;
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader];
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
None
|
||||
}
|
||||
fn discard_pending_delivery(&mut self) -> Result<(), EngineError> {
|
||||
Err(EngineError::InvalidDeliveryState)
|
||||
}
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>>;
|
||||
fn record_client_status(
|
||||
&mut self,
|
||||
client_status_code: u16,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>>;
|
||||
/// 执行已终结且结算、准入释放均已返回,协议层才可以放弃清理责任
|
||||
/// 结算失败仍由 Store 保留费用重试,释放失败仍按租约 TTL 收敛
|
||||
fn is_finalized(&self) -> bool;
|
||||
fn cancel(&self);
|
||||
/// 将会话交给宿主持续驱动清理;取消请求事件的等待不会丢弃已开始的结算
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()>;
|
||||
}
|
||||
|
||||
pub trait ExecutionService: Send + Sync {
|
||||
/// 未就绪时管理与健康接口仍可运行,模型认证按既有路径报告不可用
|
||||
fn request_settings(&self) -> Option<RequestSettings> {
|
||||
None
|
||||
}
|
||||
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError>;
|
||||
fn authenticate_request(
|
||||
&self,
|
||||
request: ClientAuthenticationRequest,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, ClientAuthenticationError>> {
|
||||
Box::pin(async move {
|
||||
let plaintext = request
|
||||
.native_bearer()
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)?;
|
||||
self.authenticate(plaintext.expose_for_auth())
|
||||
})
|
||||
}
|
||||
/// 只验证入口认证信封,不记录 Key 使用事实或执行推理准入
|
||||
fn verify_request(
|
||||
&self,
|
||||
_request: ClientAuthenticationRequest,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, ClientAuthenticationError>> {
|
||||
Box::pin(async { Err(ClientAuthenticationError::InvalidKey) })
|
||||
}
|
||||
fn public_models(&self, client: &AuthenticatedClient) -> Vec<PublicModelId>;
|
||||
fn client_model_catalog<'a>(
|
||||
&'a self,
|
||||
_client: &'a AuthenticatedClient,
|
||||
_protocol: &'a str,
|
||||
_client_version: &'a str,
|
||||
) -> BoxFuture<'a, Result<Vec<PublicModelDescriptor>, ProviderCatalogUnavailable>> {
|
||||
Box::pin(async { Err(ProviderCatalogUnavailable) })
|
||||
}
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool;
|
||||
|
||||
/// 重新鉴权并冻结本次根请求身份、请求 ID、deadline 与取消域
|
||||
fn prepare_execution(
|
||||
&self,
|
||||
client: AuthenticatedClient,
|
||||
) -> BoxFuture<'_, Result<PreparedRootExecution, GatewayError>> {
|
||||
Box::pin(async move { PreparedRootExecution::new(client) })
|
||||
}
|
||||
|
||||
/// 由已经验证管理会话与插件 models 域的宿主入口选择 Key
|
||||
/// 不向页面提供明文,也不跳过普通请求的扩展、准入和计量
|
||||
fn prepare_plugin_execution(
|
||||
&self,
|
||||
_client_key_id: &ClientApiKeyId,
|
||||
) -> BoxFuture<'_, Result<PreparedRootExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Unauthorized,
|
||||
"plugin model entry is unavailable",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// 为已经通过 `verify_request` 的只读入口建立中间件生命周期,不重复写 Key 使用事实
|
||||
fn prepare_verified_execution(
|
||||
&self,
|
||||
client: AuthenticatedClient,
|
||||
) -> Result<PreparedRootExecution, GatewayError> {
|
||||
PreparedRootExecution::new(client)
|
||||
}
|
||||
|
||||
/// 解析与准备阶段冻结的发布代次一致的中间件计划
|
||||
fn middleware_plan(&self, _prepared: &PreparedRootExecution) -> Option<FrozenMiddlewarePlan> {
|
||||
None
|
||||
}
|
||||
|
||||
/// 消费一次已准备身份并进入原有 Core 路由、准入、attempt 与结算路径
|
||||
fn start_prepared(
|
||||
&self,
|
||||
prepared: PreparedRootExecution,
|
||||
request: PreparedExecutionRequest,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
self.start(StartExecution {
|
||||
client: prepared.client,
|
||||
public_model: request.public_model,
|
||||
operation: request.operation,
|
||||
metadata: request.metadata,
|
||||
})
|
||||
.await
|
||||
})
|
||||
}
|
||||
|
||||
/// 消费一次已准备身份,并进入 Provider 自有端点的原有准入与结算路径
|
||||
fn start_prepared_provider_endpoint(
|
||||
&self,
|
||||
prepared: PreparedRootExecution,
|
||||
provider: ProviderKind,
|
||||
upstream_model: Option<UpstreamModelId>,
|
||||
operation: Operation,
|
||||
metadata: ExecutionRequestMetadata,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
self.start_provider_endpoint(StartProviderExecution {
|
||||
client: prepared.client,
|
||||
provider,
|
||||
upstream_model,
|
||||
operation,
|
||||
metadata,
|
||||
})
|
||||
.await
|
||||
})
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
|
||||
|
||||
/// 返回 Provider 注册表中首个声明 Live sideband 能力的网关。
|
||||
/// `None` 表示当前组合不含语音 sideband;协议 adapter 据此回退到稳定 501。
|
||||
fn live_gateway(&self) -> Option<Arc<dyn crate::live::LiveGateway>> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 只验证 Client API Key 并返回稳定 Key ID,不产生请求使用事实
|
||||
///
|
||||
/// 管理侧登录等非数据面场景必须使用该端口,避免把认证本身误记为一次 Key 使用
|
||||
pub trait ClientKeyVerifier: Send + Sync {
|
||||
fn verify_client_key(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<ClientApiKeyId, ClientAuthenticationError>;
|
||||
}
|
||||
|
||||
/// 成功认证后的 API Key 使用事实接收器
|
||||
///
|
||||
/// 认证仍是同步快照读取;实现必须自行异步、去重地持久化,不得阻塞客户端请求
|
||||
pub trait ClientApiKeyUsageSink: Send + Sync {
|
||||
fn record_used(&self, key_id: &ClientApiKeyId);
|
||||
}
|
||||
|
||||
pub(super) fn duration_ms(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub(super) fn new_request_id() -> Result<ModelRequestId, GatewayError> {
|
||||
ModelRequestId::new(format!("req_{}", Uuid::now_v7().simple()))
|
||||
.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "failed to allocate request ID"))
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! 执行边界的错误分类与资源释放诊断
|
||||
|
||||
use super::contract::ClientAuthenticationError;
|
||||
use crate::{
|
||||
engine::{EngineError, ModelRequestId},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
};
|
||||
|
||||
pub(super) fn authentication_gateway_error(error: ClientAuthenticationError) -> GatewayError {
|
||||
match error {
|
||||
ClientAuthenticationError::InvalidKey => {
|
||||
GatewayError::new(GatewayErrorKind::Unauthorized, "client API key is invalid")
|
||||
}
|
||||
ClientAuthenticationError::SnapshotUnavailable => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"runtime snapshot is unavailable",
|
||||
),
|
||||
ClientAuthenticationError::ProviderUnavailable => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"frontend authentication provider is unavailable",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn map_routing_error(error: crate::validation::RoutingError) -> GatewayError {
|
||||
match error {
|
||||
crate::validation::RoutingError::ModelNotFound {
|
||||
model,
|
||||
mapped_model,
|
||||
} => GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
if model == mapped_model {
|
||||
"the requested model was not found in the provider catalogs available to this API key; check the model name"
|
||||
} else {
|
||||
"the requested model maps to an upstream model that was not found in the provider catalogs available to this API key; check the configured model mapping"
|
||||
},
|
||||
),
|
||||
crate::validation::RoutingError::NoCapableProvider { .. }
|
||||
| crate::validation::RoutingError::NoCapableProviderEndpoint { .. }
|
||||
| crate::validation::RoutingError::EmptyAccountScope => GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no provider can execute this request",
|
||||
),
|
||||
crate::validation::RoutingError::UnsupportedProviderEndpoint { .. } => GatewayError::new(
|
||||
GatewayErrorKind::Unsupported,
|
||||
"the selected provider does not support this operation",
|
||||
),
|
||||
_ => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"runtime routing configuration is invalid",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
match error {
|
||||
EngineError::Cancelled => {
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
}
|
||||
EngineError::Deadline => {
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")
|
||||
}
|
||||
EngineError::Provider(provider) => GatewayError::from_provider(provider),
|
||||
EngineError::EmptyRoutingPlan | EngineError::ProviderNotRegistered { .. } => {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no provider is available",
|
||||
)
|
||||
}
|
||||
EngineError::Store(source) => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "request execution failed")
|
||||
.with_source(source.clone())
|
||||
}
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_resource_failure(
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
request_id: &ModelRequestId,
|
||||
operation: &'static str,
|
||||
message: &'static str,
|
||||
source: impl Into<crate::error::ErrorSource>,
|
||||
) {
|
||||
let mut failure = crate::diagnostics::OperationalFailure::new(
|
||||
"core",
|
||||
operation,
|
||||
"resource_unavailable",
|
||||
message,
|
||||
);
|
||||
failure.correlation_id = Some(request_id.as_str().to_owned());
|
||||
failure.details = crate::error::ErrorDetails::capture(Some(&source.into()), None, false);
|
||||
if diagnostics.record_failure(failure).await.is_err() {
|
||||
tracing::warn!(
|
||||
request_id = request_id.as_str(),
|
||||
operation,
|
||||
"resource diagnostic could not be recorded"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
//! 数据面执行入口与请求、作用域、会话的单一所有权
|
||||
|
||||
mod admission;
|
||||
mod contract;
|
||||
mod failure;
|
||||
pub use contract::{
|
||||
AuthenticatedClient, ClientApiKeyUsageSink, ClientAuthenticationError, ClientKeyVerifier,
|
||||
ClientTransport, ExecutionRequestMetadata, ExecutionService, ExecutionSession,
|
||||
PreparedExecutionRequest, PreparedRootExecution, StartExecution, StartProviderExecution,
|
||||
StartedExecution,
|
||||
};
|
||||
mod scope;
|
||||
pub use scope::BoundModelExecutionContext;
|
||||
mod service;
|
||||
pub use failure::gateway_error_from_engine;
|
||||
pub use service::DefaultExecutionService;
|
||||
mod probe_store;
|
||||
mod session;
|
||||
|
||||
use std::time::Duration;
|
||||
const DIAGNOSTIC_TIMEOUT: Duration = Duration::from_secs(10 * 60);
|
||||
const COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const MAX_NESTED_EXECUTIONS: usize = 16;
|
||||
const MAX_CONCURRENT_NESTED_EXECUTIONS: usize = 4;
|
||||
@@ -0,0 +1,64 @@
|
||||
//! 诊断探测使用的非持久执行记录与观测事实
|
||||
|
||||
use crate::{
|
||||
engine::{
|
||||
AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProviderAccountId, RecoveryReport, UpstreamSendState,
|
||||
},
|
||||
error::StoreError,
|
||||
identity::ProviderKind,
|
||||
routing::UpstreamModelId,
|
||||
};
|
||||
use std::time::SystemTime;
|
||||
|
||||
pub(super) struct ProbeObservation {
|
||||
pub(super) provider_kind: ProviderKind,
|
||||
pub(super) account_id: ProviderAccountId,
|
||||
pub(super) upstream_model: UpstreamModelId,
|
||||
}
|
||||
|
||||
pub(super) struct TransientExecutionStore;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ExecutionStore for TransientExecutionStore {
|
||||
async fn create_model_request(&self, _: NewModelRequest) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_attempt(&self, _: AttemptRecord) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
_: &ModelRequestId,
|
||||
_: UpstreamSendState,
|
||||
) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_downstream_committed(
|
||||
&self,
|
||||
_: &ModelRequestId,
|
||||
_: SystemTime,
|
||||
_: Option<u16>,
|
||||
) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_client_status(&self, _: &ModelRequestId, _: u16) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_intermediate_failure(&self, _: IntermediateFailure) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recover_expired(&self, _: SystemTime) -> Result<RecoveryReport, StoreError> {
|
||||
Ok(RecoveryReport::default())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
//! 请求嵌套图、活动权限与作用域 guard
|
||||
|
||||
use super::admission::{AdmissionLease, settle_budget};
|
||||
use super::contract::{AuthenticatedClient, ExecutionRequestMetadata};
|
||||
use super::{MAX_CONCURRENT_NESTED_EXECUTIONS, MAX_NESTED_EXECUTIONS};
|
||||
use crate::engine::budget::{ClientBudgetCharge, ClientBudgetPort};
|
||||
use crate::{
|
||||
concurrency::ConcurrencyWaitBudget,
|
||||
engine::{
|
||||
ModelRequestId, ProviderAccountId, coordinator::CoordinationExtensions,
|
||||
extensions::ExtensionCallScope, nested::ExecutionEffects,
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
identity::ProviderKind,
|
||||
lifecycle::{CancellationToken, Deadline},
|
||||
operation::Operation,
|
||||
policy::ClientApiKeyId,
|
||||
routing::{
|
||||
FrozenAccountScope, PublicModelId, UpstreamModelId, request_settings::RequestSettings,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fmt,
|
||||
sync::{
|
||||
Arc, Mutex, Weak,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
pub(super) enum ExecutionTarget {
|
||||
Model(PublicModelId),
|
||||
ProviderEndpoint {
|
||||
provider: ProviderKind,
|
||||
upstream_model: Option<UpstreamModelId>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ExecutionTarget {
|
||||
pub(super) fn public_model(&self) -> Option<&PublicModelId> {
|
||||
match self {
|
||||
Self::Model(model) => Some(model),
|
||||
Self::ProviderEndpoint { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_public_model(self) -> Option<PublicModelId> {
|
||||
match self {
|
||||
Self::Model(model) => Some(model),
|
||||
Self::ProviderEndpoint { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PendingStartExecution {
|
||||
pub(super) client: AuthenticatedClient,
|
||||
pub(super) target: ExecutionTarget,
|
||||
pub(super) operation: Operation,
|
||||
pub(super) metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
pub(super) struct AuthorizedExecution {
|
||||
pub(super) response_control: Option<crate::engine::response_control::ResponseControl>,
|
||||
pub(super) account_scope: Arc<FrozenAccountScope>,
|
||||
pub(super) deadline_at: Deadline,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) extension_scope: ExtensionCallScope,
|
||||
pub(super) required_provider: Option<ProviderKind>,
|
||||
pub(super) required_account: Option<ProviderAccountId>,
|
||||
pub(super) nested: Option<NestedExecutionFacts>,
|
||||
pub(super) bound: Option<BoundExecutionFacts>,
|
||||
pub(super) graph: Option<Arc<NestedExecutionGraph>>,
|
||||
pub(super) execution_effects_baseline: usize,
|
||||
pub(super) nested_permit: Option<NestedExecutionPermit>,
|
||||
}
|
||||
|
||||
pub(super) struct ExecutionStartGuard {
|
||||
pub(super) admission: ExecutionAdmission,
|
||||
pub(super) active_request: Option<ActiveRequestLease>,
|
||||
pub(super) concurrency_wait_budget: ConcurrencyWaitBudget,
|
||||
pub(super) admission_decision_ms: Option<u64>,
|
||||
}
|
||||
|
||||
pub(super) struct PreparedExecutionStart {
|
||||
pub(super) request_id: ModelRequestId,
|
||||
pub(super) started_at: SystemTime,
|
||||
pub(super) plan: crate::routing::RoutingPlan,
|
||||
pub(super) extensions: CoordinationExtensions,
|
||||
pub(super) authorization: AuthorizedExecution,
|
||||
pub(super) guard: ExecutionStartGuard,
|
||||
}
|
||||
|
||||
pub(super) struct NestedExecutionFacts {
|
||||
pub(super) parent_request_id: ModelRequestId,
|
||||
pub(super) initiating_plugin_instance_id: String,
|
||||
}
|
||||
|
||||
pub(super) struct BoundExecutionFacts {
|
||||
pub(super) initiating_plugin_instance_id: String,
|
||||
}
|
||||
|
||||
pub(super) struct NestedExecutionGraph {
|
||||
pub(super) total_started: AtomicUsize,
|
||||
pub(super) active: AtomicUsize,
|
||||
pub(super) effects: Arc<ExecutionEffects>,
|
||||
}
|
||||
|
||||
impl NestedExecutionGraph {
|
||||
pub(super) fn new(effects: Arc<ExecutionEffects>) -> Self {
|
||||
Self {
|
||||
total_started: AtomicUsize::new(0),
|
||||
active: AtomicUsize::new(0),
|
||||
effects,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn acquire(self: &Arc<Self>) -> Result<NestedExecutionPermit, GatewayError> {
|
||||
if self
|
||||
.active
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
||||
(active < MAX_CONCURRENT_NESTED_EXECUTIONS).then_some(active + 1)
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ConcurrencyQueueFull,
|
||||
"nested model execution concurrency is exhausted",
|
||||
));
|
||||
}
|
||||
if self
|
||||
.total_started
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |total| {
|
||||
(total < MAX_NESTED_EXECUTIONS).then_some(total + 1)
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
self.active.fetch_sub(1, Ordering::AcqRel);
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"nested model execution budget is exhausted",
|
||||
));
|
||||
}
|
||||
Ok(NestedExecutionPermit {
|
||||
graph: Arc::clone(self),
|
||||
armed: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct NestedExecutionPermit {
|
||||
pub(super) graph: Arc<NestedExecutionGraph>,
|
||||
pub(super) armed: bool,
|
||||
}
|
||||
|
||||
impl NestedExecutionPermit {
|
||||
pub(super) fn release(mut self) {
|
||||
if self.armed {
|
||||
self.graph.active.fetch_sub(1, Ordering::AcqRel);
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NestedExecutionPermit {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
self.graph.active.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ActiveRequestAuthority {
|
||||
pub(super) client: AuthenticatedClient,
|
||||
pub(super) account_scope: Arc<FrozenAccountScope>,
|
||||
pub(super) deadline_at: Deadline,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) extension_scope: ExtensionCallScope,
|
||||
pub(super) graph: Arc<NestedExecutionGraph>,
|
||||
}
|
||||
|
||||
pub(super) type ActiveRequestRegistry =
|
||||
Arc<Mutex<BTreeMap<ModelRequestId, Weak<ActiveRequestAuthority>>>>;
|
||||
|
||||
pub(super) struct ActiveRequestLease {
|
||||
pub(super) registry: ActiveRequestRegistry,
|
||||
pub(super) request_id: ModelRequestId,
|
||||
pub(super) authority: Arc<ActiveRequestAuthority>,
|
||||
}
|
||||
|
||||
/// 管理页或 CLI 一次调用持有的显式模型执行身份
|
||||
///
|
||||
/// 结构不暴露 Key 明文;Runtime 只能把它原样交回 Core 发起模型请求
|
||||
#[derive(Clone)]
|
||||
pub struct BoundModelExecutionContext {
|
||||
pub(super) authority: Arc<BoundModelExecutionAuthority>,
|
||||
}
|
||||
|
||||
impl BoundModelExecutionContext {
|
||||
#[must_use]
|
||||
pub fn request_settings(&self) -> Option<&RequestSettings> {
|
||||
self.authority.client.request_settings()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct BoundModelExecutionAuthority {
|
||||
pub(super) client: AuthenticatedClient,
|
||||
pub(super) account_scope: Arc<FrozenAccountScope>,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) extension_scope: ExtensionCallScope,
|
||||
pub(super) initiating_plugin_instance_id: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BoundModelExecutionContext {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BoundModelExecutionContext")
|
||||
.field("key_id", &self.authority.client.policy.key_id())
|
||||
.field(
|
||||
"initiating_plugin_instance_id",
|
||||
&self.authority.initiating_plugin_instance_id,
|
||||
)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BoundModelExecutionAuthority {
|
||||
fn drop(&mut self) {
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl ActiveRequestLease {
|
||||
pub(super) fn release(self) {
|
||||
drop(self);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ActiveRequestLease {
|
||||
fn drop(&mut self) {
|
||||
self.authority.cancellation.cancel();
|
||||
let mut active = self
|
||||
.registry
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if active
|
||||
.get(&self.request_id)
|
||||
.and_then(Weak::upgrade)
|
||||
.is_some_and(|authority| Arc::ptr_eq(&authority, &self.authority))
|
||||
{
|
||||
active.remove(&self.request_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionStartGuard {
|
||||
pub(super) async fn release_failed(
|
||||
self,
|
||||
budget: Option<&dyn ClientBudgetPort>,
|
||||
request_id: ModelRequestId,
|
||||
key_id: ClientApiKeyId,
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
) {
|
||||
if let Some(active_request) = self.active_request {
|
||||
active_request.release();
|
||||
}
|
||||
if let Some(budget) = budget {
|
||||
settle_budget(
|
||||
budget,
|
||||
ClientBudgetCharge {
|
||||
key_id,
|
||||
request_id,
|
||||
amount_usd: crate::metering::Decimal::ZERO,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
diagnostics,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.admission.release().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum ExecutionAdmission {
|
||||
Client(AdmissionLease),
|
||||
Nested(NestedExecutionPermit),
|
||||
Bound(AdmissionLease, NestedExecutionPermit),
|
||||
}
|
||||
|
||||
impl ExecutionAdmission {
|
||||
pub(super) async fn release(self) {
|
||||
match self {
|
||||
Self::Client(admission) => admission.release().await,
|
||||
Self::Nested(permit) => permit.release(),
|
||||
Self::Bound(admission, permit) => {
|
||||
admission.release().await;
|
||||
permit.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+90
-1196
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,213 @@
|
||||
//! 执行会话的准入释放、费用结算与终态收敛
|
||||
|
||||
use super::COORDINATION_TIMEOUT;
|
||||
use super::admission::settle_budget;
|
||||
use super::contract::ExecutionSession;
|
||||
use super::scope::{ActiveRequestLease, ExecutionAdmission};
|
||||
use crate::{
|
||||
engine::{
|
||||
CoordinatedEvent, EngineError, ExecutionStore,
|
||||
budget::ClientBudgetPort,
|
||||
continuation::{NativeContinuationPin, NativeContinuationPort},
|
||||
coordinator::ResponseExecutionSession,
|
||||
},
|
||||
event::{ProviderEvent, ProviderResponseHeader},
|
||||
operation::ProviderSessionState,
|
||||
};
|
||||
use futures::{FutureExt as _, future::BoxFuture, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) struct DefaultExecutionSession {
|
||||
pub(super) core: ResponseExecutionSession<dyn ExecutionStore>,
|
||||
pub(super) admission: Option<ExecutionAdmission>,
|
||||
pub(super) active_request: Option<ActiveRequestLease>,
|
||||
pub(super) cleanup: Option<BoxFuture<'static, ()>>,
|
||||
pub(super) continuation: Arc<dyn NativeContinuationPort>,
|
||||
pub(super) continuation_recorded: bool,
|
||||
pub(super) budget: Option<Arc<dyn ClientBudgetPort>>,
|
||||
pub(super) diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
}
|
||||
|
||||
impl DefaultExecutionSession {
|
||||
pub(super) fn new(
|
||||
core: ResponseExecutionSession<dyn ExecutionStore>,
|
||||
admission: ExecutionAdmission,
|
||||
active_request: Option<ActiveRequestLease>,
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
budget: Option<Arc<dyn ClientBudgetPort>>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
core,
|
||||
admission: Some(admission),
|
||||
active_request,
|
||||
cleanup: None,
|
||||
continuation,
|
||||
continuation_recorded: false,
|
||||
budget,
|
||||
diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn settle_if_finalized(&mut self) {
|
||||
if self.core.is_finalized()
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
if let Some(active_request) = self.active_request.take() {
|
||||
active_request.release();
|
||||
}
|
||||
let budget = self.budget.take();
|
||||
let charge = self.core.budget_charge();
|
||||
let diagnostics = self.diagnostics.clone();
|
||||
// 在首次 await 前把完整清理责任留在会话内
|
||||
// 事件等待被取消后,后续 poll
|
||||
// 或 detach 继续同一个 future,既不丢失费用,也不重启已完成的结算
|
||||
self.cleanup = Some(Box::pin(async move {
|
||||
if let Some(budget) = budget {
|
||||
settle_budget(budget.as_ref(), charge, diagnostics.as_ref()).await;
|
||||
}
|
||||
admission.release().await;
|
||||
}));
|
||||
}
|
||||
if let Some(cleanup) = self.cleanup.as_mut() {
|
||||
cleanup.await;
|
||||
self.cleanup = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_continuation(&mut self, state: Option<&ProviderSessionState>) {
|
||||
if self.continuation_recorded {
|
||||
return;
|
||||
}
|
||||
let Some(state) = state else {
|
||||
return;
|
||||
};
|
||||
let Some(pin) = self.core.native_continuation_pin(state) else {
|
||||
return;
|
||||
};
|
||||
self.continuation_recorded = true;
|
||||
record_native_continuation(self.continuation.as_ref(), pin).await;
|
||||
}
|
||||
|
||||
pub(super) async fn finalize_detached(&mut self) {
|
||||
if let Err(error) = self.core.cancel_and_finalize().await {
|
||||
tracing::warn!(request_id = self.core.request_id().as_str(), operation = "finalize_detached_execution", %error, "Detached execution 终态收敛失败");
|
||||
}
|
||||
self.settle_if_finalized().await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DefaultExecutionSession {
|
||||
fn drop(&mut self) {
|
||||
self.core.cancel();
|
||||
drop(self.active_request.take());
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionSession for DefaultExecutionSession {
|
||||
fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
self.core.trace()
|
||||
}
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
let result = self.core.next_event().await;
|
||||
if let Ok(Some(event)) = result.as_ref() {
|
||||
self.record_continuation(event.session_update()).await;
|
||||
}
|
||||
self.settle_if_finalized().await;
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
let result = self.core.collect_uncommitted().await;
|
||||
if let Ok(events) = result.as_ref() {
|
||||
let state = events.iter().find_map(ProviderEvent::session_update);
|
||||
self.record_continuation(state).await;
|
||||
}
|
||||
self.settle_if_finalized().await;
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
self.core.response_headers()
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
self.core.response_status_code()
|
||||
}
|
||||
|
||||
fn discard_pending_delivery(&mut self) -> Result<(), EngineError> {
|
||||
self.core.discard_pending_delivery()
|
||||
}
|
||||
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
let result = self.core.commit_downstream(client_status_code).await;
|
||||
self.settle_if_finalized().await;
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(
|
||||
&mut self,
|
||||
client_status_code: u16,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
let result = self.core.record_client_status(client_status_code).await;
|
||||
self.settle_if_finalized().await;
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.core.is_finalized()
|
||||
&& self.admission.is_none()
|
||||
&& self.active_request.is_none()
|
||||
&& self.cleanup.is_none()
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
self.core.cancel();
|
||||
}
|
||||
|
||||
fn detach_finalize(mut self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move { self.finalize_detached().await })
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let provider = pin.provider().as_str().to_owned();
|
||||
let account = pin.account().as_str().to_owned();
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
%error,
|
||||
"Continuation affinity 写入失败,后续请求将退化为外部续接"
|
||||
);
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
"Continuation affinity 后台写入超时,已丢弃本次亲和记录"
|
||||
);
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 插件嵌套调用的递归作用域
|
||||
//! 执行扩展计划的弱引用索引、冻结解析与插件递归作用域
|
||||
|
||||
use std::{collections::BTreeSet, sync::Arc};
|
||||
|
||||
@@ -43,3 +43,82 @@ impl ExtensionCallScope {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 同一代次的具体执行计划;不拥有发布状态,运行资源由对应发布集合保活
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ExecutionExtensionPlans {
|
||||
pub middleware: Option<Arc<dyn super::middleware::MiddlewarePlan>>,
|
||||
pub upstream_adapters: Option<Arc<dyn super::upstream_adapter::UpstreamAdapterPlan>>,
|
||||
}
|
||||
|
||||
/// 同一发布代次不能被静默替换
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("execution extension generation is already registered")]
|
||||
pub struct ExtensionRegistrationError;
|
||||
|
||||
/// 仅以弱引用按冻结身份解析执行计划,不延长旧代次寿命或决定何时发布
|
||||
#[derive(Clone, Default)]
|
||||
pub struct ExecutionExtensionIndex {
|
||||
sets: Arc<
|
||||
std::sync::RwLock<
|
||||
std::collections::BTreeMap<
|
||||
crate::routing::extensions::ExtensionSetId,
|
||||
std::sync::Weak<ExecutionExtensionPlans>,
|
||||
>,
|
||||
>,
|
||||
>,
|
||||
}
|
||||
|
||||
impl ExecutionExtensionIndex {
|
||||
pub fn register(
|
||||
&self,
|
||||
id: crate::routing::extensions::ExtensionSetId,
|
||||
plans: Arc<ExecutionExtensionPlans>,
|
||||
) -> Result<Arc<ExecutionExtensionPlans>, ExtensionRegistrationError> {
|
||||
let mut sets = self
|
||||
.sets
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
sets.retain(|_, plans| plans.strong_count() > 0);
|
||||
if sets.contains_key(&id) {
|
||||
return Err(ExtensionRegistrationError);
|
||||
}
|
||||
sets.insert(id, Arc::downgrade(&plans));
|
||||
Ok(plans)
|
||||
}
|
||||
|
||||
fn resolve(
|
||||
&self,
|
||||
generation: &crate::routing::extensions::ExtensionSetReference,
|
||||
) -> Option<Arc<ExecutionExtensionPlans>> {
|
||||
self.sets
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(generation.id())
|
||||
.and_then(std::sync::Weak::upgrade)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn middleware(
|
||||
&self,
|
||||
generation: &crate::routing::extensions::ExtensionSetReference,
|
||||
) -> Option<super::middleware::FrozenMiddlewarePlan> {
|
||||
let plan = self.resolve(generation)?.middleware.clone()?;
|
||||
Some(super::middleware::FrozenMiddlewarePlan::new(
|
||||
plan,
|
||||
generation.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_adapters(
|
||||
&self,
|
||||
generation: &crate::routing::extensions::ExtensionSetReference,
|
||||
) -> Option<super::upstream_adapter::FrozenUpstreamAdapterPlan> {
|
||||
let plan = self.resolve(generation)?.upstream_adapters.clone()?;
|
||||
Some(super::upstream_adapter::FrozenUpstreamAdapterPlan::new(
|
||||
plan,
|
||||
generation.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
+5
-177
@@ -5,10 +5,11 @@
|
||||
//! Runtime/SDK 负责;身份、路由、账号租约、发送事实、重试、计量与结算仍由 Core
|
||||
//! 持有,不能通过本端口交给插件解释
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use super::FrozenMiddlewarePlan;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::{Arc, RwLock, Weak};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
@@ -27,8 +28,7 @@ use crate::operation::{
|
||||
CapabilityRequirements, Feature, GenerateRequest, Operation, OperationKind, ProtocolPayload,
|
||||
};
|
||||
use crate::policy::ClientApiKeyId;
|
||||
use crate::runtime::extensions::{ExtensionSetId, ExtensionSetReference};
|
||||
use crate::settings::RequestSettings;
|
||||
use crate::routing::request_settings::RequestSettings;
|
||||
|
||||
/// 中间件在一次逻辑请求中的挂载位置
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
@@ -543,7 +543,7 @@ impl fmt::Debug for MiddlewareResponseEnvelope {
|
||||
/// Core 冻结的调用事实;完整业务身份可由插件读取
|
||||
#[derive(Clone)]
|
||||
pub struct MiddlewareContext {
|
||||
plan: Option<FrozenMiddlewarePlan>,
|
||||
pub(super) plan: Option<FrozenMiddlewarePlan>,
|
||||
request_id: ModelRequestId,
|
||||
mount: MiddlewareMount,
|
||||
attempt_index: Option<NonZeroU32>,
|
||||
@@ -799,175 +799,3 @@ impl From<ProviderError> for MiddlewareError {
|
||||
/// 绑定当前调用且只能消费一次的宿主续体
|
||||
pub type MiddlewareNext =
|
||||
crate::middleware::Next<MiddlewareRequest, MiddlewareResponse, MiddlewareError>;
|
||||
|
||||
/// 一个发布代次冻结的统一数据面中间件计划
|
||||
pub trait MiddlewarePlan: Send + Sync + fmt::Debug {
|
||||
fn has_service(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn handle_service(
|
||||
&self,
|
||||
_context: crate::middleware::service::Context,
|
||||
input: crate::middleware::service::Value,
|
||||
next: crate::middleware::service::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<crate::middleware::service::Value, crate::middleware::service::Error>,
|
||||
> {
|
||||
next.run(input)
|
||||
}
|
||||
|
||||
fn has_websocket(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn handle_websocket(
|
||||
&self,
|
||||
_context: crate::middleware::websocket::Context,
|
||||
message: crate::middleware::websocket::Message,
|
||||
next: crate::middleware::websocket::Next,
|
||||
) -> BoxFuture<'static, Result<Option<crate::middleware::websocket::Message>, MiddlewareError>>
|
||||
{
|
||||
next.run(message)
|
||||
}
|
||||
|
||||
/// HTTP 入口在认证、路由与正文解析之前组合,无匹配时保留直接路径
|
||||
fn has_http(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn handle_http(
|
||||
&self,
|
||||
_context: crate::middleware::http::Context,
|
||||
request: crate::middleware::http::Request,
|
||||
next: crate::middleware::http::Next,
|
||||
) -> BoxFuture<'static, Result<crate::middleware::http::Response, MiddlewareError>> {
|
||||
next.run(request)
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
context: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<MiddlewareResponse, MiddlewareError>>;
|
||||
}
|
||||
|
||||
/// 调用期间同时保活发布代次与中间件计划
|
||||
#[derive(Clone)]
|
||||
pub struct FrozenMiddlewarePlan {
|
||||
plan: Arc<dyn MiddlewarePlan>,
|
||||
_generation: ExtensionSetReference,
|
||||
}
|
||||
|
||||
impl FrozenMiddlewarePlan {
|
||||
#[must_use]
|
||||
pub fn has_service(&self) -> bool {
|
||||
self.plan.has_service()
|
||||
}
|
||||
pub fn handle_service(
|
||||
&self,
|
||||
context: crate::middleware::service::Context,
|
||||
input: crate::middleware::service::Value,
|
||||
next: crate::middleware::service::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<crate::middleware::service::Value, crate::middleware::service::Error>,
|
||||
> {
|
||||
self.plan.handle_service(context, input, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_websocket(&self) -> bool {
|
||||
self.plan.has_websocket()
|
||||
}
|
||||
pub fn handle_websocket(
|
||||
&self,
|
||||
context: crate::middleware::websocket::Context,
|
||||
message: crate::middleware::websocket::Message,
|
||||
next: crate::middleware::websocket::Next,
|
||||
) -> BoxFuture<'static, Result<Option<crate::middleware::websocket::Message>, MiddlewareError>>
|
||||
{
|
||||
self.plan.handle_websocket(context, message, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_http(&self) -> bool {
|
||||
self.plan.has_http()
|
||||
}
|
||||
|
||||
pub fn handle_http(
|
||||
&self,
|
||||
mut context: crate::middleware::http::Context,
|
||||
request: crate::middleware::http::Request,
|
||||
next: crate::middleware::http::Next,
|
||||
) -> BoxFuture<'static, Result<crate::middleware::http::Response, MiddlewareError>> {
|
||||
context.plan = Some(self.clone());
|
||||
self.plan.handle_http(context, request, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(plan: Arc<dyn MiddlewarePlan>, generation: ExtensionSetReference) -> Self {
|
||||
Self {
|
||||
plan,
|
||||
_generation: generation,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn handle(
|
||||
&self,
|
||||
mut context: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<MiddlewareResponse, MiddlewareError>> {
|
||||
context.plan = Some(self.clone());
|
||||
self.plan.handle(context, request, next)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for FrozenMiddlewarePlan {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("FrozenMiddlewarePlan")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// 同一发布代次不能被静默替换
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("middleware generation is already registered")]
|
||||
pub struct MiddlewareRegistrationError;
|
||||
|
||||
/// 按请求冻结的发布集合解析中间件计划;索引不延长旧代次寿命
|
||||
#[derive(Clone, Default)]
|
||||
pub struct MiddlewareExtensionIndex {
|
||||
sets: Arc<RwLock<BTreeMap<ExtensionSetId, Weak<dyn MiddlewarePlan>>>>,
|
||||
}
|
||||
|
||||
impl MiddlewareExtensionIndex {
|
||||
pub fn register(
|
||||
&self,
|
||||
id: ExtensionSetId,
|
||||
plan: Arc<dyn MiddlewarePlan>,
|
||||
) -> Result<Arc<dyn MiddlewarePlan>, MiddlewareRegistrationError> {
|
||||
let mut sets = self
|
||||
.sets
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
sets.retain(|_, plan| plan.strong_count() > 0);
|
||||
if sets.contains_key(&id) {
|
||||
return Err(MiddlewareRegistrationError);
|
||||
}
|
||||
sets.insert(id, Arc::downgrade(&plan));
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn resolve(&self, generation: &ExtensionSetReference) -> Option<FrozenMiddlewarePlan> {
|
||||
self.sets
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(generation.id())
|
||||
.and_then(Weak::upgrade)
|
||||
.map(|plan| FrozenMiddlewarePlan::new(plan, generation.clone()))
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -13,7 +13,7 @@ use crate::{engine::middleware::MiddlewareError, lifecycle::CancellationToken};
|
||||
pub type Body = UnsyncBoxBody<Bytes, Box<dyn std::error::Error + Send + Sync>>;
|
||||
pub type Request = http::Request<Body>;
|
||||
pub type Response = http::Response<Body>;
|
||||
pub type Next = super::Next<Request, Response, MiddlewareError>;
|
||||
pub type Next = crate::middleware::Next<Request, Response, MiddlewareError>;
|
||||
|
||||
/// 路由前调用没有 Client Key;身份在默认认证流程中产生
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -29,7 +29,7 @@ pub struct Context {
|
||||
|
||||
/// 内部子请求复用 API 总路由;端口不解释业务路径,也不经网络回环
|
||||
pub trait Dispatcher: Send + Sync {
|
||||
fn request_settings(&self) -> Option<crate::settings::RequestSettings> {
|
||||
fn request_settings(&self) -> Option<crate::routing::request_settings::RequestSettings> {
|
||||
None
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ pub trait Dispatcher: Send + Sync {
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Settings {
|
||||
pub timeout: Option<Duration>,
|
||||
pub runtime: Option<crate::settings::RequestSettings>,
|
||||
pub runtime: Option<crate::routing::request_settings::RequestSettings>,
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
+1
-1
@@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex};
|
||||
|
||||
use futures::future::{BoxFuture, Shared};
|
||||
|
||||
use crate::{engine::middleware::MiddlewareError, middleware::websocket::Session};
|
||||
use crate::{engine::middleware::MiddlewareError, engine::middleware::websocket::Session};
|
||||
|
||||
pub type PendingSession =
|
||||
Shared<BoxFuture<'static, Result<Arc<dyn Session>, Arc<MiddlewareError>>>>;
|
||||
@@ -0,0 +1,15 @@
|
||||
//! 执行层的请求、HTTP、WebSocket 与公开服务中间件合同
|
||||
|
||||
mod contract;
|
||||
pub mod http;
|
||||
mod plan;
|
||||
pub mod service;
|
||||
pub mod websocket;
|
||||
|
||||
pub use contract::{
|
||||
MiddlewareAuthority, MiddlewareBody, MiddlewareCapabilityDeclaration, MiddlewareContext,
|
||||
MiddlewareError, MiddlewareFrame, MiddlewareFrameEnvelope, MiddlewareFraming, MiddlewareHeader,
|
||||
MiddlewareMount, MiddlewareNext, MiddlewareRequest, MiddlewareResponse,
|
||||
MiddlewareResponseEnvelope, MiddlewareTarget,
|
||||
};
|
||||
pub use plan::{FrozenMiddlewarePlan, MiddlewarePlan};
|
||||
@@ -0,0 +1,152 @@
|
||||
//! 与发布代次一起冻结的协议及请求中间件计划
|
||||
|
||||
use super::{
|
||||
MiddlewareContext, MiddlewareError, MiddlewareNext, MiddlewareRequest, MiddlewareResponse,
|
||||
};
|
||||
use crate::routing::extensions::ExtensionSetReference;
|
||||
use futures::future::BoxFuture;
|
||||
use std::{fmt, sync::Arc};
|
||||
|
||||
/// 一个发布代次冻结的统一数据面中间件计划
|
||||
pub trait MiddlewarePlan: Send + Sync + fmt::Debug {
|
||||
fn has_service(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn handle_service(
|
||||
&self,
|
||||
_context: crate::engine::middleware::service::Context,
|
||||
input: crate::engine::middleware::service::Value,
|
||||
next: crate::engine::middleware::service::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<
|
||||
crate::engine::middleware::service::Value,
|
||||
crate::engine::middleware::service::Error,
|
||||
>,
|
||||
> {
|
||||
next.run(input)
|
||||
}
|
||||
|
||||
fn has_websocket(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn handle_websocket(
|
||||
&self,
|
||||
_context: crate::engine::middleware::websocket::Context,
|
||||
message: crate::engine::middleware::websocket::Message,
|
||||
next: crate::engine::middleware::websocket::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<Option<crate::engine::middleware::websocket::Message>, MiddlewareError>,
|
||||
> {
|
||||
next.run(message)
|
||||
}
|
||||
|
||||
/// HTTP 入口在认证、路由与正文解析之前组合,无匹配时保留直接路径
|
||||
fn has_http(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn handle_http(
|
||||
&self,
|
||||
_context: crate::engine::middleware::http::Context,
|
||||
request: crate::engine::middleware::http::Request,
|
||||
next: crate::engine::middleware::http::Next,
|
||||
) -> BoxFuture<'static, Result<crate::engine::middleware::http::Response, MiddlewareError>>
|
||||
{
|
||||
next.run(request)
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
context: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<MiddlewareResponse, MiddlewareError>>;
|
||||
}
|
||||
|
||||
/// 调用期间同时保活发布代次与中间件计划
|
||||
#[derive(Clone)]
|
||||
pub struct FrozenMiddlewarePlan {
|
||||
plan: Arc<dyn MiddlewarePlan>,
|
||||
_generation: ExtensionSetReference,
|
||||
}
|
||||
|
||||
impl FrozenMiddlewarePlan {
|
||||
#[must_use]
|
||||
pub fn has_service(&self) -> bool {
|
||||
self.plan.has_service()
|
||||
}
|
||||
pub fn handle_service(
|
||||
&self,
|
||||
context: crate::engine::middleware::service::Context,
|
||||
input: crate::engine::middleware::service::Value,
|
||||
next: crate::engine::middleware::service::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<
|
||||
crate::engine::middleware::service::Value,
|
||||
crate::engine::middleware::service::Error,
|
||||
>,
|
||||
> {
|
||||
self.plan.handle_service(context, input, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_websocket(&self) -> bool {
|
||||
self.plan.has_websocket()
|
||||
}
|
||||
pub fn handle_websocket(
|
||||
&self,
|
||||
context: crate::engine::middleware::websocket::Context,
|
||||
message: crate::engine::middleware::websocket::Message,
|
||||
next: crate::engine::middleware::websocket::Next,
|
||||
) -> BoxFuture<
|
||||
'static,
|
||||
Result<Option<crate::engine::middleware::websocket::Message>, MiddlewareError>,
|
||||
> {
|
||||
self.plan.handle_websocket(context, message, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_http(&self) -> bool {
|
||||
self.plan.has_http()
|
||||
}
|
||||
|
||||
pub fn handle_http(
|
||||
&self,
|
||||
mut context: crate::engine::middleware::http::Context,
|
||||
request: crate::engine::middleware::http::Request,
|
||||
next: crate::engine::middleware::http::Next,
|
||||
) -> BoxFuture<'static, Result<crate::engine::middleware::http::Response, MiddlewareError>>
|
||||
{
|
||||
context.plan = Some(self.clone());
|
||||
self.plan.handle_http(context, request, next)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(plan: Arc<dyn MiddlewarePlan>, generation: ExtensionSetReference) -> Self {
|
||||
Self {
|
||||
plan,
|
||||
_generation: generation,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn handle(
|
||||
&self,
|
||||
mut context: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
) -> BoxFuture<'static, Result<MiddlewareResponse, MiddlewareError>> {
|
||||
context.plan = Some(self.clone());
|
||||
self.plan.handle(context, request, next)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for FrozenMiddlewarePlan {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("FrozenMiddlewarePlan")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -7,7 +7,7 @@ use crate::{
|
||||
|
||||
/// 值只在进入插件链时编码;操作的领域校验与副作用由终端所属服务执行
|
||||
pub type Value = serde_json::Value;
|
||||
pub type Next = super::Next<Value, Value, Error>;
|
||||
pub type Next = crate::middleware::Next<Value, Value, Error>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
+1
-1
@@ -44,4 +44,4 @@ pub struct Context {
|
||||
pub cancellation: CancellationToken,
|
||||
pub sender: Arc<dyn Sender>,
|
||||
}
|
||||
pub type Next = super::Next<Message, Option<Message>, MiddlewareError>;
|
||||
pub type Next = crate::middleware::Next<Message, Option<Message>, MiddlewareError>;
|
||||
@@ -101,17 +101,7 @@ pub enum AttemptTrigger {
|
||||
AccountRetry,
|
||||
}
|
||||
|
||||
/// Provider 可为当前 attempt 选择的请求局部传输档位
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
pub enum AttemptTransport {
|
||||
/// 使用 Provider 的默认传输策略
|
||||
#[default]
|
||||
Default,
|
||||
/// 固定账号重试 Provider 的首选传输;序号由 Provider 的独立预算驱动
|
||||
Retry(NonZeroU32),
|
||||
/// 使用 Provider 定义的备用传输
|
||||
Fallback,
|
||||
}
|
||||
pub use crate::upstream::AttemptTransport;
|
||||
|
||||
impl AttemptTrigger {
|
||||
#[must_use]
|
||||
|
||||
@@ -65,7 +65,7 @@ pub struct NestedModelExecutionRequest {
|
||||
/// Core 在绑定时冻结当前
|
||||
/// Key 策略;每个实际模型请求仍独立经过准入、预算、账本与计费
|
||||
pub struct BoundModelExecutionBinding {
|
||||
pub settings: Option<crate::settings::RequestSettings>,
|
||||
pub settings: Option<crate::routing::request_settings::RequestSettings>,
|
||||
pub client_key_id: ClientApiKeyId,
|
||||
pub initiating_plugin_instance_id: String,
|
||||
/// 身份随父调用或连接回收;每个模型请求使用独立的执行期限
|
||||
|
||||
@@ -22,8 +22,8 @@ use crate::{
|
||||
metering::{CostEstimate, CostSource, Usage},
|
||||
operation::OperationKind,
|
||||
policy::ClientApiKeyId,
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
routing::{AccountGroupId, ConfigRevision, PublicModelId},
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
upstream::UpstreamSendState,
|
||||
};
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ use crate::{
|
||||
identity::ProviderKind,
|
||||
operation::Operation,
|
||||
policy::ClientApiKeyId,
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
routing::{AccountGroupId, PublicModelId},
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
};
|
||||
|
||||
/// 模型路由插件的一次输入;正文仍保存在 `operation` 中,由 Runtime 按权限投影
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
metering::{CalculatedCost, Usage},
|
||||
operation::Operation,
|
||||
routing::UpstreamModelId,
|
||||
runtime::extensions::ExtensionSetReference,
|
||||
routing::extensions::ExtensionSetReference,
|
||||
};
|
||||
|
||||
use super::{
|
||||
|
||||
@@ -144,7 +144,7 @@ pub enum PreDeliveryRetry {
|
||||
/// 可靠 NotSent 的建连恢复;次数与时间由 Core 的请求级预算裁决
|
||||
SameAccountConnectionRetry {
|
||||
/// Provider 明确指定实际失败的传输,避免 HTTP 恢复重新进入 WS
|
||||
transport: crate::engine::AttemptTransport,
|
||||
transport: crate::upstream::AttemptTransport,
|
||||
},
|
||||
/// 固定本次账号,并要求 Provider 使用备用传输
|
||||
SameAccountTransportFallback,
|
||||
@@ -666,7 +666,7 @@ impl ProviderError {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_connection_retry(mut self, transport: crate::engine::AttemptTransport) -> Self {
|
||||
pub fn with_connection_retry(mut self, transport: crate::upstream::AttemptTransport) -> Self {
|
||||
self.pre_delivery_retry = Some(Box::new(PreDeliveryRetry::SameAccountConnectionRetry {
|
||||
transport,
|
||||
}));
|
||||
|
||||
@@ -128,7 +128,7 @@ pub struct CoreStartup {
|
||||
providers: ProviderRegistry,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
frontend_authentication: Option<engine::authentication::FrontendAuthenticationExtensionIndex>,
|
||||
control: CoreControlPlaneStartup,
|
||||
}
|
||||
@@ -164,7 +164,7 @@ impl CoreStartup {
|
||||
self.providers,
|
||||
self.request_observers,
|
||||
self.request_policies,
|
||||
self.middlewares,
|
||||
self.execution_extensions,
|
||||
self.frontend_authentication,
|
||||
));
|
||||
let execution: Arc<dyn ExecutionService> = service.clone();
|
||||
@@ -194,7 +194,7 @@ fn build_execution_service(
|
||||
providers: ProviderRegistry,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
frontend_authentication: Option<engine::authentication::FrontendAuthenticationExtensionIndex>,
|
||||
) -> DefaultExecutionService {
|
||||
let mut service = DefaultExecutionService::new(
|
||||
@@ -215,8 +215,8 @@ fn build_execution_service(
|
||||
if let Some(request_policies) = request_policies {
|
||||
service = service.with_request_policies(request_policies);
|
||||
}
|
||||
if let Some(middlewares) = middlewares {
|
||||
service = service.with_middlewares(middlewares);
|
||||
if let Some(execution_extensions) = execution_extensions {
|
||||
service = service.with_execution_extensions(execution_extensions);
|
||||
}
|
||||
if let Some(frontend_authentication) = frontend_authentication {
|
||||
service = service.with_frontend_authentication(frontend_authentication);
|
||||
@@ -276,10 +276,10 @@ impl CoreBundle {
|
||||
pub async fn initialize(
|
||||
ports: CoreStorePorts,
|
||||
providers: ProviderRegistry,
|
||||
extensions: Option<Arc<dyn runtime::extensions::ExtensionPreparationPort>>,
|
||||
extensions: Option<Arc<dyn routing::extensions::ExtensionPreparationPort>>,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
frontend_authentication: Option<engine::authentication::FrontendAuthenticationExtensionIndex>,
|
||||
) -> Result<CoreBundle, CoreError> {
|
||||
prepare(
|
||||
@@ -288,7 +288,7 @@ pub async fn initialize(
|
||||
extensions,
|
||||
request_observers,
|
||||
request_policies,
|
||||
middlewares,
|
||||
execution_extensions,
|
||||
frontend_authentication,
|
||||
)
|
||||
.activate()
|
||||
@@ -300,10 +300,10 @@ pub async fn initialize(
|
||||
pub fn prepare(
|
||||
ports: CoreStorePorts,
|
||||
providers: ProviderRegistry,
|
||||
extensions: Option<Arc<dyn runtime::extensions::ExtensionPreparationPort>>,
|
||||
extensions: Option<Arc<dyn routing::extensions::ExtensionPreparationPort>>,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
frontend_authentication: Option<engine::authentication::FrontendAuthenticationExtensionIndex>,
|
||||
) -> CoreStartup {
|
||||
let control = prepare_control_plane(ports.clone(), providers.clone(), extensions);
|
||||
@@ -312,7 +312,7 @@ pub fn prepare(
|
||||
providers,
|
||||
request_observers,
|
||||
request_policies,
|
||||
middlewares,
|
||||
execution_extensions,
|
||||
frontend_authentication,
|
||||
control,
|
||||
}
|
||||
@@ -377,7 +377,7 @@ pub struct CoreCommandPlaneStartup {
|
||||
providers: ProviderRegistry,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
control: CoreControlPlaneStartup,
|
||||
}
|
||||
|
||||
@@ -407,7 +407,7 @@ impl CoreCommandPlaneStartup {
|
||||
self.providers,
|
||||
self.request_observers,
|
||||
self.request_policies,
|
||||
self.middlewares,
|
||||
self.execution_extensions,
|
||||
None,
|
||||
)
|
||||
.with_snapshot_refresh(Arc::clone(&control.publisher)),
|
||||
@@ -449,10 +449,10 @@ impl CoreCommandPlaneBundle {
|
||||
pub fn prepare_command_plane(
|
||||
ports: CoreStorePorts,
|
||||
providers: ProviderRegistry,
|
||||
extensions: Option<Arc<dyn runtime::extensions::ExtensionPreparationPort>>,
|
||||
extensions: Option<Arc<dyn routing::extensions::ExtensionPreparationPort>>,
|
||||
request_observers: Option<engine::observation::RequestObserverExtensionIndex>,
|
||||
request_policies: Option<engine::policy::RequestPolicyExtensionIndex>,
|
||||
middlewares: Option<engine::middleware::MiddlewareExtensionIndex>,
|
||||
execution_extensions: Option<engine::extensions::ExecutionExtensionIndex>,
|
||||
) -> CoreCommandPlaneStartup {
|
||||
let control = prepare_control_plane(ports.clone(), providers.clone(), extensions);
|
||||
CoreCommandPlaneStartup {
|
||||
@@ -460,7 +460,7 @@ pub fn prepare_command_plane(
|
||||
providers,
|
||||
request_observers,
|
||||
request_policies,
|
||||
middlewares,
|
||||
execution_extensions,
|
||||
control,
|
||||
}
|
||||
}
|
||||
@@ -470,7 +470,7 @@ pub fn prepare_command_plane(
|
||||
pub fn prepare_control_plane(
|
||||
ports: CoreStorePorts,
|
||||
providers: ProviderRegistry,
|
||||
extensions: Option<Arc<dyn runtime::extensions::ExtensionPreparationPort>>,
|
||||
extensions: Option<Arc<dyn routing::extensions::ExtensionPreparationPort>>,
|
||||
) -> CoreControlPlaneStartup {
|
||||
let mut compiler = RuntimeSnapshotCompiler::new(ports.snapshots, Arc::new(providers));
|
||||
if let Some(extensions) = extensions {
|
||||
|
||||
-4
@@ -1,9 +1,5 @@
|
||||
//! 与协议无关的洋葱组合器;输入、结果与错误保留调用方的领域类型
|
||||
|
||||
pub mod http;
|
||||
pub mod service;
|
||||
pub mod websocket;
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
/// 单次调用中的一层;上下文放在类型化输入中或由闭包捕获
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
//! Provider 会话亲和、绑定和排除事实
|
||||
|
||||
use super::{ProviderStoreError, ProviderStoreErrorKind};
|
||||
use crate::{account::ProviderAccountId, identity::ProviderKind};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{collections::BTreeSet, fmt, time::Duration};
|
||||
|
||||
/// Provider 从原始会话锚点派生的不可逆亲和键
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct ProviderSessionAffinityKey(String);
|
||||
|
||||
impl ProviderSessionAffinityKey {
|
||||
pub fn try_new(value: impl Into<String>) -> Result<Self, ProviderStoreError> {
|
||||
let value = value.into();
|
||||
if value.is_empty()
|
||||
|| value.len() > 128
|
||||
|| !value.bytes().all(|byte| {
|
||||
byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'-' | b'_')
|
||||
})
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate provider session affinity key",
|
||||
));
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn expose_to_store(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderSessionAffinityKey {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ProviderSessionAffinityKey([OPAQUE])")
|
||||
}
|
||||
}
|
||||
|
||||
/// 会话绑定快照;版本区分同一账号的不同认领,防止过期和 A → B → A 后的旧写入
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderSessionBinding {
|
||||
account_id: ProviderAccountId,
|
||||
revision: String,
|
||||
}
|
||||
|
||||
impl ProviderSessionBinding {
|
||||
pub fn new(
|
||||
account_id: ProviderAccountId,
|
||||
revision: String,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if revision.len() != 32 || !revision.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"decode provider session binding revision",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
account_id,
|
||||
revision,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn revision(&self) -> &str {
|
||||
&self.revision
|
||||
}
|
||||
}
|
||||
|
||||
/// 已观测请求关联及其账号迁移权限,由 Provider 解释协议后写入
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderSessionAlias {
|
||||
pub session_key: ProviderSessionAffinityKey,
|
||||
pub root_session_key: Option<ProviderSessionAffinityKey>,
|
||||
pub follow_only: bool,
|
||||
}
|
||||
|
||||
/// 客户端作用域内的会话账号绑定,原始会话身份由 Provider 哈希后传入
|
||||
pub trait ProviderSessionAffinityPort: Send + Sync {
|
||||
fn load<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionBinding>, ProviderStoreError>>;
|
||||
|
||||
/// 在发送前原子认领、续期或迁移;None 表示快照已变化,调用方必须释放租约重新选择
|
||||
/// expected 为 None 只允许首次认领,已有快照必须连同版本匹配
|
||||
fn compare_and_bind<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
expected: Option<&'a ProviderSessionBinding>,
|
||||
account_id: &'a ProviderAccountId,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionBinding>, ProviderStoreError>>;
|
||||
|
||||
/// 显式观测到的请求关联只指向会话键,不缓存账号,迁移后仍读取当前绑定
|
||||
fn load_alias<'a>(
|
||||
&'a self,
|
||||
provider: &'a ProviderKind,
|
||||
alias: &'a ProviderSessionAffinityKey,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionAlias>, ProviderStoreError>>;
|
||||
|
||||
/// 只允许首次关联或同目标续期,冲突时禁止把同一轮次改指其他会话
|
||||
fn bind_alias<'a>(
|
||||
&'a self,
|
||||
provider: &'a ProviderKind,
|
||||
alias: &'a ProviderSessionAffinityKey,
|
||||
session: &'a ProviderSessionAlias,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Provider 会话内已失败账号的可丢失排除集
|
||||
///
|
||||
/// Provider 自行派生会话键并决定何时写入或清理;Core 只承载调度所需的账号 ID
|
||||
/// 与 compare-and-swap revision,不解释任一 Provider 协议字段
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderSessionExclusions {
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
revision: String,
|
||||
}
|
||||
|
||||
impl ProviderSessionExclusions {
|
||||
#[must_use]
|
||||
pub const fn new(excluded_accounts: BTreeSet<ProviderAccountId>, revision: String) -> Self {
|
||||
Self {
|
||||
excluded_accounts,
|
||||
revision,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn excluded_accounts(&self) -> &BTreeSet<ProviderAccountId> {
|
||||
&self.excluded_accounts
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn revision(&self) -> &str {
|
||||
&self.revision
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的 Provider 会话级账号排除状态
|
||||
///
|
||||
/// 该端口不接收协议正文;Provider 只能以不可逆会话键、账号 ID 和固定 TTL 操作
|
||||
pub trait ProviderSessionExclusionPort: Send + Sync {
|
||||
fn load<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderSessionExclusions>, ProviderStoreError>>;
|
||||
|
||||
fn record_failure<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
account_id: &'a ProviderAccountId,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<ProviderSessionExclusions, ProviderStoreError>>;
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
key: &'a ProviderSessionAffinityKey,
|
||||
expected_revision: &'a str,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
//! 按能力提供 Provider 所需的存储端口
|
||||
|
||||
use super::{
|
||||
OAuthPendingFlowPort, ProviderArtifactProfileCachePort, ProviderCatalogCachePort,
|
||||
ProviderCooldownPort, ProviderCredentialStatePort, ProviderLeasePort,
|
||||
ProviderRuntimePolicyPort, ProviderSessionAffinityPort, ProviderSessionExclusionPort,
|
||||
};
|
||||
use crate::account::{AccountFeedbackStats, ProviderAccountStore};
|
||||
use std::{fmt, sync::Arc};
|
||||
|
||||
/// Provider 只能按能力取用端口,无法取得 Redis client 或 repository 集合
|
||||
#[derive(Clone)]
|
||||
pub struct ProviderStorePorts {
|
||||
accounts: Arc<dyn ProviderAccountStore>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
session_affinity: Arc<dyn ProviderSessionAffinityPort>,
|
||||
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
|
||||
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
oauth_pending: Arc<dyn OAuthPendingFlowPort>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
}
|
||||
|
||||
impl ProviderStorePorts {
|
||||
#[must_use]
|
||||
// 每个参数代表独立能力端口;合并为单一配置对象会隐藏 Provider 能力边界
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
accounts: Arc<dyn ProviderAccountStore>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
session_affinity: Arc<dyn ProviderSessionAffinityPort>,
|
||||
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
|
||||
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
|
||||
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
oauth_pending: Arc<dyn OAuthPendingFlowPort>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
leases,
|
||||
session_affinity,
|
||||
session_exclusions,
|
||||
account_feedback: Arc::new(AccountFeedbackStats::default()),
|
||||
catalog_cache,
|
||||
artifact_profiles,
|
||||
credential_state,
|
||||
cooldowns,
|
||||
runtime_policy,
|
||||
oauth_pending,
|
||||
diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn diagnostics(&self) -> Arc<dyn crate::diagnostics::OperationalDiagnostics> {
|
||||
self.diagnostics.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn accounts(&self) -> Arc<dyn ProviderAccountStore> {
|
||||
Arc::clone(&self.accounts)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn leases(&self) -> Arc<dyn ProviderLeasePort> {
|
||||
Arc::clone(&self.leases)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_affinity(&self) -> Arc<dyn ProviderSessionAffinityPort> {
|
||||
Arc::clone(&self.session_affinity)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_exclusions(&self) -> Arc<dyn ProviderSessionExclusionPort> {
|
||||
Arc::clone(&self.session_exclusions)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn account_feedback(&self) -> Arc<AccountFeedbackStats> {
|
||||
Arc::clone(&self.account_feedback)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn catalog_cache(&self) -> Arc<dyn ProviderCatalogCachePort> {
|
||||
Arc::clone(&self.catalog_cache)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn artifact_profiles(&self) -> Arc<dyn ProviderArtifactProfileCachePort> {
|
||||
Arc::clone(&self.artifact_profiles)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn credential_state(&self) -> Arc<dyn ProviderCredentialStatePort> {
|
||||
Arc::clone(&self.credential_state)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn cooldowns(&self) -> Arc<dyn ProviderCooldownPort> {
|
||||
Arc::clone(&self.cooldowns)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn runtime_policy(&self) -> Arc<dyn ProviderRuntimePolicyPort> {
|
||||
Arc::clone(&self.runtime_policy)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn oauth_pending(&self) -> Arc<dyn OAuthPendingFlowPort> {
|
||||
Arc::clone(&self.oauth_pending)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderStorePorts {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ProviderStorePorts([CAPABILITIES])")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Provider 目录与身份画像缓存端口
|
||||
|
||||
use super::ProviderStoreError;
|
||||
use crate::{
|
||||
account::OpaqueProviderData,
|
||||
identity::ProviderKind,
|
||||
validation::{IdentifierError, validate_text},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{
|
||||
fmt,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
/// Provider 定义的 catalog cache 作用域
|
||||
///
|
||||
/// Core 不解释其值;例如 Provider 可以使用套餐、区域或产品线作为共享目录边界
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct ProviderCatalogScope(String);
|
||||
|
||||
impl ProviderCatalogScope {
|
||||
/// 创建一个稳定且可用于 Redis 隔离键的 Provider-owned 作用域
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空值、过长文本或控制字符会被拒绝
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, 128, false, None)?;
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
/// 返回 Provider-owned 作用域文本
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque catalog cache 的 Provider 与 Provider-owned 作用域
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCatalogCacheKey {
|
||||
provider_kind: ProviderKind,
|
||||
scope: ProviderCatalogScope,
|
||||
}
|
||||
|
||||
impl ProviderCatalogCacheKey {
|
||||
#[must_use]
|
||||
pub const fn new(provider_kind: ProviderKind, scope: ProviderCatalogScope) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
scope,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &ProviderCatalogScope {
|
||||
&self.scope
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCatalogCachePort: Send + Sync {
|
||||
fn replace<'a>(
|
||||
&'a self,
|
||||
key: &'a ProviderCatalogCacheKey,
|
||||
catalog: &'a OpaqueProviderData,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
key: &'a ProviderCatalogCacheKey,
|
||||
) -> BoxFuture<'a, Result<Option<OpaqueProviderData>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Provider 从官方制品核验出的可重建请求画像
|
||||
///
|
||||
/// Core 只用单调制品序号约束覆盖顺序;具体版本字段由对应 Provider 放在
|
||||
/// `profile` 中解释
|
||||
/// 每个 Provider 的各制品分别保留一份最新画像
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct ProviderArtifactProfile {
|
||||
provider_kind: ProviderKind,
|
||||
artifact_key: String,
|
||||
artifact_sequence: u64,
|
||||
verified_at: SystemTime,
|
||||
profile: OpaqueProviderData,
|
||||
}
|
||||
|
||||
impl ProviderArtifactProfile {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
provider_kind: ProviderKind,
|
||||
artifact_key: String,
|
||||
artifact_sequence: u64,
|
||||
verified_at: SystemTime,
|
||||
profile: OpaqueProviderData,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
artifact_key,
|
||||
artifact_sequence,
|
||||
verified_at,
|
||||
profile,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn artifact_key(&self) -> &str {
|
||||
&self.artifact_key
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn artifact_sequence(&self) -> u64 {
|
||||
self.artifact_sequence
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn verified_at(&self) -> SystemTime {
|
||||
self.verified_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn profile(&self) -> &OpaqueProviderData {
|
||||
&self.profile
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderArtifactProfile {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderArtifactProfile")
|
||||
.field("provider_kind", &self.provider_kind)
|
||||
.field("artifact_sequence", &self.artifact_sequence)
|
||||
.field("verified_at", &self.verified_at)
|
||||
.field("profile", &"[PROVIDER-OWNED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderArtifactProfileCachePort: Send + Sync {
|
||||
/// 覆盖同一 Provider、同一制品的固定 cache key
|
||||
///
|
||||
/// 返回 `false` 表示 Store 已持有更高的制品序号;相同序号但内容不同必须返回
|
||||
/// [`super::ProviderStoreErrorKind::Conflict`],不能静默改写已核验画像
|
||||
fn replace_if_newer(
|
||||
&self,
|
||||
profile: ProviderArtifactProfile,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
artifact_key: &'a str,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderArtifactProfile>, ProviderStoreError>>;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
//! Provider 冷却、容量窗口和运行状态端口
|
||||
|
||||
use super::ProviderStoreError;
|
||||
/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理
|
||||
pub use crate::account::AccountCooldownKind as ProviderCooldownKind;
|
||||
use crate::{
|
||||
account::{CredentialRevision, ProviderAccountId},
|
||||
routing::UpstreamModelId,
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
}
|
||||
|
||||
impl ProviderCooldown {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self::new_with_kind(
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
ProviderCooldownKind::RateLimit,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new_with_kind(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scheduling_state(&self) -> crate::account::AccountCooldown {
|
||||
crate::account::AccountCooldown {
|
||||
until: self.until,
|
||||
kind: self.kind,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderCooldownKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum ProviderCooldownScope {
|
||||
/// 只阻止同一账号调用指定上游模型
|
||||
UpstreamModel(UpstreamModelId),
|
||||
}
|
||||
|
||||
impl ProviderCooldownScope {
|
||||
#[must_use]
|
||||
pub const fn upstream_model(model: UpstreamModelId) -> Self {
|
||||
Self::UpstreamModel(model)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UpstreamModel(_) => "model",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn value(&self) -> &str {
|
||||
match self {
|
||||
Self::UpstreamModel(model) => model.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不进入账号持久状态的账号+作用域 cooldown
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderScopedCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
}
|
||||
|
||||
impl ProviderScopedCooldown {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
scope,
|
||||
until,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &ProviderCooldownScope {
|
||||
&self.scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCooldownPort: Send + Sync {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>>;
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>;
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key
|
||||
/// 生命周期与 credential revision 无关;不要求调用方持有 revision
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号
|
||||
/// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)
|
||||
/// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
|
||||
/// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>>;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
//! Provider 凭据运行状态读写端口
|
||||
|
||||
use super::ProviderStoreError;
|
||||
use crate::account::{CredentialRevision, CredentialState, ProviderAccountId};
|
||||
use futures::future::BoxFuture;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
/// Redis 中可重建的账号状态投影
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCredentialState {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
}
|
||||
|
||||
impl ProviderCredentialState {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
enabled,
|
||||
credential_state,
|
||||
observed_at,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_state(&self) -> CredentialState {
|
||||
self.credential_state
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn observed_at(&self) -> SystemTime {
|
||||
self.observed_at
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
fn replace(
|
||||
&self,
|
||||
state: ProviderCredentialState,
|
||||
) -> BoxFuture<'_, Result<(), ProviderStoreError>>;
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCredentialState>, ProviderStoreError>>;
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次瞬态刷新失败并返回窗口内累计失败次数;每次失败刷新 TTL
|
||||
fn record_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 凭据完整成功轮换后清零失败计数,退避窗口重新从 base 起步
|
||||
fn clear_refresh_backoff<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
//! Provider 存储端口的错误分类与原始来源
|
||||
|
||||
/// Provider 可据此决定是否重试,但看不到 SQL、Redis 或秘密原文
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProviderStoreErrorKind {
|
||||
Unavailable,
|
||||
InvalidData,
|
||||
Conflict,
|
||||
}
|
||||
|
||||
/// Provider 存储端口的脱敏错误
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("provider store {operation} failed: {kind:?}")]
|
||||
pub struct ProviderStoreError {
|
||||
kind: ProviderStoreErrorKind,
|
||||
operation: &'static str,
|
||||
source: Option<crate::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl ProviderStoreError {
|
||||
#[must_use]
|
||||
pub const fn new(kind: ProviderStoreErrorKind, operation: &'static str) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
operation,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 包装基础设施失败,分类与原始来源分别保留
|
||||
#[must_use]
|
||||
pub fn caused_by(
|
||||
kind: ProviderStoreErrorKind,
|
||||
operation: &'static str,
|
||||
source: impl Into<crate::error::ErrorSource>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
operation,
|
||||
source: Some(source.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderStoreErrorKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
//! Provider 调度与刷新租约的中立端口
|
||||
|
||||
use super::ProviderStoreError;
|
||||
use crate::{
|
||||
account::{AccountConcurrency, AccountRuntimeSignals, CredentialRevision, ProviderAccountId},
|
||||
identity::ProviderKind,
|
||||
policy::ClientApiKeyId,
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{collections::BTreeMap, fmt, num::NonZeroU32, time::Duration};
|
||||
|
||||
/// 一个 Provider 的完整可重建调度状态
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderSchedulingState {
|
||||
signals: BTreeMap<ProviderAccountId, AccountRuntimeSignals>,
|
||||
round_robin_cursor: u64,
|
||||
}
|
||||
|
||||
impl ProviderSchedulingState {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
signals: BTreeMap<ProviderAccountId, AccountRuntimeSignals>,
|
||||
round_robin_cursor: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
signals,
|
||||
round_robin_cursor,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn signals(&self) -> &BTreeMap<ProviderAccountId, AccountRuntimeSignals> {
|
||||
&self.signals
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn round_robin_cursor(&self) -> u64 {
|
||||
self.round_robin_cursor
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求级账号 lease 的全部中立事实
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProviderSchedulingLeaseRequest {
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
max_concurrent: AccountConcurrency,
|
||||
request_interval: Duration,
|
||||
deadline: crate::lifecycle::Deadline,
|
||||
cancellation: crate::lifecycle::CancellationToken,
|
||||
}
|
||||
|
||||
impl ProviderSchedulingLeaseRequest {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
max_concurrent: impl Into<AccountConcurrency>,
|
||||
request_interval: Duration,
|
||||
deadline: impl Into<crate::lifecycle::Deadline>,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
account_id,
|
||||
credential_revision,
|
||||
max_concurrent: max_concurrent.into(),
|
||||
request_interval,
|
||||
deadline: deadline.into(),
|
||||
cancellation: crate::lifecycle::CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_cancellation(mut self, cancellation: crate::lifecycle::CancellationToken) -> Self {
|
||||
self.cancellation = cancellation;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn cancellation(&self) -> crate::lifecycle::CancellationToken {
|
||||
self.cancellation.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn max_concurrent(&self) -> AccountConcurrency {
|
||||
self.max_concurrent
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_interval(&self) -> Duration {
|
||||
self.request_interval
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn deadline(&self) -> crate::lifecycle::Deadline {
|
||||
self.deadline
|
||||
}
|
||||
}
|
||||
|
||||
/// Lease 生命周期由具体 Store guard 管理,Provider 只能持有
|
||||
pub trait ProviderLeaseGuard: Send + Sync + 'static {}
|
||||
|
||||
impl<T> ProviderLeaseGuard for T where T: Send + Sync + 'static {}
|
||||
|
||||
pub enum ProviderLeaseAcquisition {
|
||||
Acquired(Box<dyn ProviderLeaseGuard>),
|
||||
Busy { retry_after: Option<Duration> },
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderLeaseAcquisition {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Acquired(_) => formatter.write_str("Acquired([LEASE])"),
|
||||
Self::Busy { retry_after } => formatter
|
||||
.debug_struct("Busy")
|
||||
.field("retry_after", retry_after)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 运行时会持有的三类 lease;刷新必须同时持有全局容量与账号互斥 lease
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ProviderLeaseRequest {
|
||||
Scheduling(ProviderSchedulingLeaseRequest),
|
||||
RefreshCapacity(ProviderRefreshCapacityRequest),
|
||||
Refresh(ProviderRefreshLeaseRequest),
|
||||
}
|
||||
|
||||
pub trait ProviderLeasePort: Send + Sync {
|
||||
fn load_state<'a>(
|
||||
&'a self,
|
||||
client_api_key_id: &'a ClientApiKeyId,
|
||||
provider_kind: &'a ProviderKind,
|
||||
accounts: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<ProviderSchedulingState, ProviderStoreError>>;
|
||||
|
||||
fn try_acquire(
|
||||
&self,
|
||||
request: ProviderLeaseRequest,
|
||||
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>>;
|
||||
|
||||
/// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据,
|
||||
/// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)
|
||||
fn account_in_flight<'a>(
|
||||
&'a self,
|
||||
account_ids: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<BTreeMap<ProviderAccountId, u32>, ProviderStoreError>> {
|
||||
let _ = account_ids;
|
||||
Box::pin(async move { Ok(BTreeMap::new()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// 所有 Provider 共享的 OAuth refresh 并发容量
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderRefreshCapacityRequest {
|
||||
max_concurrent: NonZeroU32,
|
||||
}
|
||||
|
||||
impl ProviderRefreshCapacityRequest {
|
||||
#[must_use]
|
||||
pub const fn new(max_concurrent: NonZeroU32) -> Self {
|
||||
Self { max_concurrent }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn max_concurrent(self) -> NonZeroU32 {
|
||||
self.max_concurrent
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderRefreshLeaseRequest {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
}
|
||||
|
||||
impl ProviderRefreshLeaseRequest {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
//! Provider 运行时所需的存储合同,按能力组织端口与共享策略
|
||||
|
||||
mod error;
|
||||
pub use error::{ProviderStoreError, ProviderStoreErrorKind};
|
||||
mod lease;
|
||||
pub use lease::{
|
||||
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderSchedulingLeaseRequest,
|
||||
ProviderSchedulingState,
|
||||
};
|
||||
mod affinity;
|
||||
pub use affinity::{
|
||||
ProviderSessionAffinityKey, ProviderSessionAffinityPort, ProviderSessionAlias,
|
||||
ProviderSessionBinding, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
};
|
||||
mod catalog;
|
||||
pub use catalog::{
|
||||
ProviderArtifactProfile, ProviderArtifactProfileCachePort, ProviderCatalogCacheKey,
|
||||
ProviderCatalogCachePort, ProviderCatalogScope,
|
||||
};
|
||||
mod credential;
|
||||
pub use credential::{ProviderCredentialState, ProviderCredentialStatePort};
|
||||
mod cooldown;
|
||||
pub use cooldown::{
|
||||
ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope,
|
||||
ProviderScopedCooldown,
|
||||
};
|
||||
mod refresh;
|
||||
pub use refresh::{ProviderRefreshPolicy, provider_refresh_backoff_at, provider_refresh_retry_at};
|
||||
mod runtime_policy;
|
||||
pub use runtime_policy::{
|
||||
ProviderFreezePolicy, ProviderRuntimePolicyPort, ProviderWarmupPolicy,
|
||||
valid_warmup_schedule_time,
|
||||
};
|
||||
mod oauth;
|
||||
pub use oauth::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome,
|
||||
};
|
||||
mod bundle;
|
||||
pub use bundle::ProviderStorePorts;
|
||||
@@ -0,0 +1,184 @@
|
||||
//! OAuth 授权 pending flow 的领取与消费合同
|
||||
|
||||
use super::{ProviderStoreError, ProviderStoreErrorKind};
|
||||
use crate::{account::OpaqueProviderData, identity::ProviderKind};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{fmt, time::Duration};
|
||||
|
||||
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
/// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct OAuthPendingBinding(String);
|
||||
|
||||
impl OAuthPendingBinding {
|
||||
pub fn try_new(value: impl Into<String>) -> Result<Self, ProviderStoreError> {
|
||||
let value = value.into();
|
||||
if value.is_empty() || value.len() > 512 {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate OAuth pending binding",
|
||||
));
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn expose_to_store(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for OAuthPendingBinding {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("OAuthPendingBinding([REDACTED])")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct NewOAuthPendingFlow {
|
||||
provider_kind: ProviderKind,
|
||||
flow: OAuthPendingBinding,
|
||||
owner: OAuthPendingBinding,
|
||||
ttl: Duration,
|
||||
payload: OpaqueProviderData,
|
||||
}
|
||||
|
||||
impl NewOAuthPendingFlow {
|
||||
pub fn try_new(
|
||||
provider_kind: ProviderKind,
|
||||
flow: OAuthPendingBinding,
|
||||
owner: OAuthPendingBinding,
|
||||
ttl: Duration,
|
||||
payload: OpaqueProviderData,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if ttl.is_zero() || ttl > MAX_PENDING_FLOW_TTL {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate OAuth pending TTL",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
provider_kind,
|
||||
flow,
|
||||
owner,
|
||||
ttl,
|
||||
payload,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn flow(&self) -> &OAuthPendingBinding {
|
||||
&self.flow
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn owner(&self) -> &OAuthPendingBinding {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn ttl(&self) -> Duration {
|
||||
self.ttl
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn payload(&self) -> &OpaqueProviderData {
|
||||
&self.payload
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for NewOAuthPendingFlow {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("NewOAuthPendingFlow")
|
||||
.field("provider_kind", &self.provider_kind)
|
||||
.field("flow", &self.flow)
|
||||
.field("owner", &self.owner)
|
||||
.field("ttl", &self.ttl)
|
||||
.field("payload", &"[PROVIDER-OWNED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingPutOutcome {
|
||||
Stored,
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
/// OAuth 回调处理取得临时 flow 的独占权结果
|
||||
///
|
||||
/// 与一次性消费不同,Provider 在上游换取令牌失败时可以释放 claim,让同一 flow
|
||||
/// 使用新的回调地址重试;只有完整校验成功后才会消费 flow
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub enum OAuthPendingClaimOutcome {
|
||||
Claimed(OpaqueProviderData),
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
InProgress,
|
||||
}
|
||||
|
||||
impl fmt::Debug for OAuthPendingClaimOutcome {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Claimed(_) => formatter.write_str("Claimed([PROVIDER-OWNED])"),
|
||||
Self::NotFound => formatter.write_str("NotFound"),
|
||||
Self::OwnerMismatch => formatter.write_str("OwnerMismatch"),
|
||||
Self::InProgress => formatter.write_str("InProgress"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingReleaseOutcome {
|
||||
Released,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingConsumeOutcome {
|
||||
Consumed,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
pub trait OAuthPendingFlowPort: Send + Sync {
|
||||
fn put_if_absent(
|
||||
&self,
|
||||
flow: NewOAuthPendingFlow,
|
||||
) -> BoxFuture<'_, Result<OAuthPendingPutOutcome, ProviderStoreError>>;
|
||||
|
||||
fn claim_if_owner<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
claim_ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingClaimOutcome, ProviderStoreError>>;
|
||||
|
||||
fn release_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingReleaseOutcome, ProviderStoreError>>;
|
||||
|
||||
fn consume_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingConsumeOutcome, ProviderStoreError>>;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Provider 刷新时机、退避与重试策略
|
||||
|
||||
use super::{ProviderStoreError, ProviderStoreErrorKind};
|
||||
use crate::account::ProviderAccountId;
|
||||
use std::{
|
||||
num::NonZeroU32,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderRefreshPolicy {
|
||||
margin: Duration,
|
||||
concurrency: NonZeroU32,
|
||||
}
|
||||
|
||||
impl ProviderRefreshPolicy {
|
||||
pub fn try_new(margin: Duration, concurrency: NonZeroU32) -> Result<Self, ProviderStoreError> {
|
||||
if margin.is_zero() {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate refresh policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
margin,
|
||||
concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn margin(self) -> Duration {
|
||||
self.margin
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn concurrency(self) -> NonZeroU32 {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 判断 AT 是否已经进入当前配置的提前刷新窗口
|
||||
///
|
||||
/// `margin` 是运行时策略,不投影为账号的持久化时间字段;已过期 AT 也需要
|
||||
/// 尝试用 RT 恢复,因此视为到期
|
||||
#[must_use]
|
||||
pub fn is_refresh_due(
|
||||
self,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> bool {
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= self.margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 按账号派生 `[0, margin]` 内的整秒错峰偏移。
|
||||
///
|
||||
/// 网关以固定周期扫描到期账号,到期时刻相同的账号会在同一轮齐刷,
|
||||
/// 在 auth.openai.com 侧形成单 IP 批量刷新特征。偏移从账号 ID 的
|
||||
/// 稳定哈希派生(与退避扰动同源、独立 salt),跨扫描与重启保持不变;
|
||||
/// 只会增大有效提前量,不会比配置的 margin 更贴近过期时刻。
|
||||
#[must_use]
|
||||
pub fn refresh_stagger(self, account_id: &ProviderAccountId) -> Duration {
|
||||
// margin 以整秒配置;秒级粒度在默认 300s 下对应约 10 个扫描槽位。
|
||||
let bound = u32::try_from(self.margin.as_secs()).unwrap_or(u32::MAX);
|
||||
Duration::from_secs(u64::from(stable_factor(
|
||||
account_id.as_str(),
|
||||
"refresh-stagger",
|
||||
0,
|
||||
bound,
|
||||
)))
|
||||
}
|
||||
|
||||
/// 含账号错峰偏移的到期判定;扫描路径先按 [`Self::staggered_refresh_bound`]
|
||||
/// 取回候选超集,再用它在内存中收窄到本轮真正到期的账号。
|
||||
#[must_use]
|
||||
pub fn is_refresh_due_staggered(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> bool {
|
||||
let staggered_margin = self.margin.saturating_add(self.refresh_stagger(account_id));
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= staggered_margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 错峰候选窗口上界;覆盖 margin 与最大偏移之和。
|
||||
///
|
||||
/// 与 [`Self::refresh_stagger`] 的值域同址维护:偏移上限为 margin,
|
||||
/// 因此 `2 × margin` 必然覆盖最大有效提前量。
|
||||
#[must_use]
|
||||
pub fn staggered_refresh_bound(self) -> Duration {
|
||||
self.margin.saturating_mul(2)
|
||||
}
|
||||
}
|
||||
|
||||
/// 指数退避基准延迟;attempt=1 即为该值
|
||||
const REFRESH_BACKOFF_BASE_DELAY: Duration = Duration::from_secs(5);
|
||||
/// 每多一次连续失败,基准延迟乘以该因子
|
||||
const REFRESH_BACKOFF_FACTOR: u32 = 3;
|
||||
/// 退避延迟上限,避免连续失败时无限增长
|
||||
const REFRESH_BACKOFF_CAP: Duration = Duration::from_secs(300);
|
||||
|
||||
/// 基于连续失败计数的指数退避重试时刻;复用 `provider_refresh_retry_at` 的稳定扰动
|
||||
///
|
||||
/// `attempt` 为窗口内累计失败次数(0 与 1 等价,均取基准延迟)
|
||||
/// 延迟按
|
||||
/// `base * factor^(attempt-1)` 增长并封顶到 `REFRESH_BACKOFF_CAP`
|
||||
pub fn provider_refresh_backoff_at(
|
||||
account_id: &ProviderAccountId,
|
||||
observed_at: SystemTime,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let exponent = attempt.saturating_sub(1);
|
||||
let multiplier = REFRESH_BACKOFF_FACTOR.saturating_pow(exponent);
|
||||
let scaled_seconds = REFRESH_BACKOFF_BASE_DELAY
|
||||
.as_secs()
|
||||
.saturating_mul(u64::from(multiplier))
|
||||
.min(REFRESH_BACKOFF_CAP.as_secs());
|
||||
let base_delay = Duration::from_secs(scaled_seconds);
|
||||
provider_refresh_retry_at(account_id, observed_at, base_delay, reason)
|
||||
}
|
||||
|
||||
/// 临时失败后的持久重试时刻;稳定扰动避免多实例同频重试
|
||||
pub fn provider_refresh_retry_at(
|
||||
account_id: &ProviderAccountId,
|
||||
observed_at: SystemTime,
|
||||
base_delay: Duration,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
if base_delay.is_zero() {
|
||||
return Err(invalid_refresh_policy("schedule refresh retry"));
|
||||
}
|
||||
let factor = stable_factor(account_id.as_str(), reason, 800, 1_200);
|
||||
let millis = u64::try_from(base_delay.as_millis())
|
||||
.unwrap_or(u64::MAX)
|
||||
.saturating_mul(u64::from(factor))
|
||||
.saturating_add(500)
|
||||
/ 1_000;
|
||||
observed_at
|
||||
.checked_add(Duration::from_millis(millis.max(1)))
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule refresh retry"))
|
||||
}
|
||||
|
||||
fn stable_factor(value: &str, salt: &str, minimum: u32, maximum: u32) -> u32 {
|
||||
let mut hash = 0xcbf2_9ce4_8422_2325_u64;
|
||||
for byte in value.bytes().chain([0]).chain(salt.bytes()) {
|
||||
hash ^= u64::from(byte);
|
||||
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
|
||||
}
|
||||
let width = u64::from(maximum - minimum) + 1;
|
||||
minimum + u32::try_from(hash % width).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn invalid_refresh_policy(operation: &'static str) -> ProviderStoreError {
|
||||
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! Provider 运行策略端口及冻结、预热配置
|
||||
|
||||
use super::{ProviderRefreshPolicy, ProviderStoreError, ProviderStoreErrorKind};
|
||||
use crate::{account::OpaqueProviderData, identity::ProviderKind, routing::ConfigRevision};
|
||||
use futures::future::BoxFuture;
|
||||
use std::time::Duration;
|
||||
|
||||
pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
/// 原子推进预热执行游标,重启或时钟回拨后不重复领取已消费的时刻
|
||||
fn claim_warmup_slot<'a>(
|
||||
&'a self,
|
||||
_timezone: crate::time::DeploymentTimeZone,
|
||||
_slot: chrono::NaiveDateTime,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async {
|
||||
Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::Unavailable,
|
||||
"claim warmup slot",
|
||||
))
|
||||
})
|
||||
}
|
||||
/// 仅首次启动写入该 Provider 的默认选择;已保存的管理配置始终优先
|
||||
fn initialize_request_profile<'a>(
|
||||
&'a self,
|
||||
_provider: &'a ProviderKind,
|
||||
initial: OpaqueProviderData,
|
||||
) -> BoxFuture<'a, Result<OpaqueProviderData, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(initial) })
|
||||
}
|
||||
|
||||
/// 读取候选配置版本实际引用的全局与 Client Key 画像配置
|
||||
///
|
||||
/// 实现必须在同一数据库快照内核对 revision,且只返回画像投影,不能读取 Key
|
||||
/// 明文
|
||||
/// Provider 代次据此在发布前拒绝已失效的选择
|
||||
fn load_request_profile_configurations<'a>(
|
||||
&'a self,
|
||||
_revision: ConfigRevision,
|
||||
_provider: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<Vec<OpaqueProviderData>, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(Vec::new()) })
|
||||
}
|
||||
|
||||
fn load_refresh_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>>;
|
||||
|
||||
/// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) })
|
||||
}
|
||||
|
||||
/// 读取账号模型预激活策略;默认关闭,只有实现运行时设置的存储需要覆盖
|
||||
fn load_warmup_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderWarmupPolicy, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(ProviderWarmupPolicy::disabled()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`,
|
||||
/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderFreezePolicy {
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window: Duration,
|
||||
freeze_duration: Duration,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl ProviderFreezePolicy {
|
||||
/// 管理写入与运行策略构造共用的冻结约束
|
||||
pub fn validate_values(
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
duration_seconds: u64,
|
||||
probe_model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(
|
||||
(2..=1_000).contains(&threshold),
|
||||
"account_auto_freeze_threshold",
|
||||
),
|
||||
(
|
||||
(60..=3_600).contains(&window_seconds),
|
||||
"account_auto_freeze_window_seconds",
|
||||
),
|
||||
(
|
||||
(300..=604_800).contains(&duration_seconds),
|
||||
"account_auto_freeze_duration_seconds",
|
||||
),
|
||||
(
|
||||
valid_optional_probe_model(probe_model),
|
||||
"account_auto_freeze_probe_model",
|
||||
),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
|
||||
/// store 层写入前已校验,这里兜底防御越界配置
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
freeze_duration_seconds: u64,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if Self::validate_values(
|
||||
threshold,
|
||||
window_seconds,
|
||||
freeze_duration_seconds,
|
||||
probe_model.as_deref(),
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate freeze policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
threshold,
|
||||
window: Duration::from_secs(window_seconds),
|
||||
freeze_duration: Duration::from_secs(freeze_duration_seconds),
|
||||
probe_enabled,
|
||||
probe_model,
|
||||
adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
/// 功能关闭时的全零策略;触发路径与 worker 都以此短路
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
threshold: 2,
|
||||
window: Duration::from_secs(60),
|
||||
freeze_duration: Duration::from_secs(300),
|
||||
probe_enabled: false,
|
||||
probe_model: None,
|
||||
adaptive_concurrency: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
/// 窗口内触发冻结的请求级失败次数阈值
|
||||
#[must_use]
|
||||
pub const fn threshold(&self) -> u32 {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
/// 失败计数滑动窗口;每次失败都会顺延窗口
|
||||
#[must_use]
|
||||
pub const fn window(&self) -> Duration {
|
||||
self.window
|
||||
}
|
||||
|
||||
/// 冻结时长;探测失败后的顺延也使用该值
|
||||
#[must_use]
|
||||
pub const fn freeze_duration(&self) -> Duration {
|
||||
self.freeze_duration
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn probe_enabled(&self) -> bool {
|
||||
self.probe_enabled
|
||||
}
|
||||
|
||||
/// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型
|
||||
#[must_use]
|
||||
pub fn probe_model(&self) -> Option<&str> {
|
||||
self.probe_model.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn adaptive_concurrency(&self) -> bool {
|
||||
self.adaptive_concurrency
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验每日预激活时间格式,如 "08:00" 或 "08:00,13:00"
|
||||
#[must_use]
|
||||
pub fn valid_warmup_schedule_time(value: &str) -> bool {
|
||||
if value.is_empty()
|
||||
|| value.len() > 255
|
||||
|| value != value.trim()
|
||||
|| value.chars().any(char::is_control)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
for part in value.split(',') {
|
||||
let bytes = part.as_bytes();
|
||||
if bytes.len() != 5 || bytes[2] != b':' {
|
||||
return false;
|
||||
}
|
||||
let Ok(hour) = part[0..2].parse::<u32>() else {
|
||||
return false;
|
||||
};
|
||||
let Ok(minute) = part[3..5].parse::<u32>() else {
|
||||
return false;
|
||||
};
|
||||
if hour > 23 || minute > 59 {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// 账号模型预激活(预热)策略;来源于 `runtime_settings`
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderWarmupPolicy {
|
||||
enabled: bool,
|
||||
schedule_time: String,
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
impl ProviderWarmupPolicy {
|
||||
/// 保存设置与创建预热策略使用相同的时间表和模型约束
|
||||
pub fn validate_values(
|
||||
enabled: bool,
|
||||
schedule_time: &str,
|
||||
model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
if !valid_warmup_schedule_time(schedule_time) {
|
||||
return Err("account_warmup_schedule_time");
|
||||
}
|
||||
if (enabled && model.is_none()) || !valid_optional_probe_model(model) {
|
||||
return Err("account_warmup_model");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
schedule_time: String,
|
||||
model: Option<String>,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if Self::validate_values(enabled, &schedule_time, model.as_deref()).is_err() {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate warmup policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
schedule_time,
|
||||
model,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
schedule_time: String::new(),
|
||||
model: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn schedule_time(&self) -> &str {
|
||||
&self.schedule_time
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn model(&self) -> Option<&str> {
|
||||
self.model.as_deref()
|
||||
}
|
||||
|
||||
/// 解析每日时间列表,返回如 `vec![(8, 0)]`
|
||||
#[must_use]
|
||||
pub fn scheduled_times(&self) -> Vec<(u32, u32)> {
|
||||
self.schedule_time
|
||||
.split(',')
|
||||
.filter_map(|part| {
|
||||
let part = part.trim();
|
||||
let mut iter = part.split(':');
|
||||
let hour = iter.next()?.parse::<u32>().ok()?;
|
||||
let minute = iter.next()?.parse::<u32>().ok()?;
|
||||
Some((hour, minute))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_optional_probe_model(model: Option<&str>) -> bool {
|
||||
model.is_none_or(|model| {
|
||||
!model.is_empty()
|
||||
&& model.len() <= 128
|
||||
&& model == model.trim()
|
||||
&& !model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
-16
@@ -32,13 +32,6 @@ impl ExtensionSetId {
|
||||
pub trait ExtensionSetLease: Send + Sync {
|
||||
fn is_ready(&self) -> bool;
|
||||
|
||||
/// 与发布集合一起冻结的上游适配计划,不能在请求中解析为新代次
|
||||
fn upstream_adapters(
|
||||
&self,
|
||||
) -> Option<Arc<dyn crate::engine::upstream_adapter::UpstreamAdapterPlan>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// 与集合一起冻结的目录事实,读取不调用插件进程
|
||||
fn model_aliases(&self) -> &[ContributedModelAlias] {
|
||||
&[]
|
||||
@@ -57,15 +50,6 @@ pub struct ExtensionSetReference {
|
||||
}
|
||||
|
||||
impl ExtensionSetReference {
|
||||
#[must_use]
|
||||
pub fn upstream_adapters(
|
||||
&self,
|
||||
) -> Option<crate::engine::upstream_adapter::FrozenUpstreamAdapterPlan> {
|
||||
self.lease.upstream_adapters().map(|plan| {
|
||||
crate::engine::upstream_adapter::FrozenUpstreamAdapterPlan::new(plan, self.clone())
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(id: ExtensionSetId, lease: Arc<dyn ExtensionSetLease>) -> Self {
|
||||
Self { id, lease }
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Provider、模型目录、精确模型映射与请求级候选计划
|
||||
|
||||
mod catalog;
|
||||
pub mod extensions;
|
||||
pub mod request_settings;
|
||||
pub mod snapshot;
|
||||
|
||||
pub use crate::account::scope::{
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
//! 冻结宿主基线与请求显式改写;派生快照只属于当前调用,不发布全局配置
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::account::FastMode;
|
||||
use crate::policy::{ClientApiKeyId, ClientPolicy, ClientSettings, RateLimits};
|
||||
use crate::routing::RuntimeSnapshot;
|
||||
use crate::settings::{InvalidSettings, SettingsValues};
|
||||
|
||||
/// 一次模型调用的有效设置;改写只影响当前请求,不发布配置或修改持久化 revision
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionSettings {
|
||||
pub runtime: SettingsValues,
|
||||
pub fast_mode: FastMode,
|
||||
pub client_limits: RateLimits,
|
||||
/// null 表示不限制总时长;显式时限从请求开始计时,改写不重置计时原点
|
||||
pub timeout_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SettingOverride<T = Value> {
|
||||
pub instance_id: String,
|
||||
pub order: u64,
|
||||
pub value: T,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
struct ExecutionOverrides {
|
||||
#[serde(skip)]
|
||||
defaults: Arc<ClientSettings>,
|
||||
client_key_id: String,
|
||||
input: Arc<ExecutionSettings>,
|
||||
fast_mode: Option<SettingOverride<FastMode>>,
|
||||
client_limits: Option<SettingOverride<RateLimits>>,
|
||||
timeout_ms: Option<SettingOverride<Option<u64>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
struct HttpTimeout {
|
||||
input_ms: Option<u64>,
|
||||
change: Option<SettingOverride<Option<u64>>>,
|
||||
}
|
||||
|
||||
/// 共享值均不可变;同级子调用的修改不会回写父调用
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RequestSettings {
|
||||
baseline: Arc<RuntimeSnapshot>,
|
||||
effective: Arc<RuntimeSnapshot>,
|
||||
overrides: Arc<BTreeMap<String, SettingOverride>>,
|
||||
execution: Option<Arc<ExecutionOverrides>>,
|
||||
http_timeout: Option<HttpTimeout>,
|
||||
order: u64,
|
||||
}
|
||||
|
||||
impl PartialEq for RequestSettings {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
Arc::ptr_eq(&self.baseline, &other.baseline)
|
||||
&& self.overrides == other.overrides
|
||||
&& self.execution == other.execution
|
||||
&& self.http_timeout == other.http_timeout
|
||||
}
|
||||
}
|
||||
impl Eq for RequestSettings {}
|
||||
|
||||
impl RequestSettings {
|
||||
#[must_use]
|
||||
pub fn new(snapshot: Arc<RuntimeSnapshot>) -> Self {
|
||||
Self {
|
||||
baseline: snapshot.clone(),
|
||||
effective: snapshot,
|
||||
overrides: Arc::default(),
|
||||
execution: None,
|
||||
http_timeout: None,
|
||||
order: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> Arc<RuntimeSnapshot> {
|
||||
self.effective.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn values(&self) -> &SettingsValues {
|
||||
self.effective.settings()
|
||||
}
|
||||
|
||||
/// 来源与有效值分开查询,不允许插件伪造其他实例的写入来源
|
||||
pub fn inspect(&self) -> Value {
|
||||
serde_json::json!({
|
||||
"config_revision": self.baseline.revision().get(),
|
||||
"host": self.baseline.settings(),
|
||||
"overrides": self.overrides,
|
||||
"execution": self.execution,
|
||||
"http_timeout": self.http_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn replace(
|
||||
&self,
|
||||
values: SettingsValues,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
self.replace_scoped(self.values(), values, instance_id)
|
||||
}
|
||||
|
||||
/// Key 画像等隐式作用域结果不记录成插件改写,避免随后切换 Key 时泄漏
|
||||
fn replace_scoped(
|
||||
&self,
|
||||
previous: &SettingsValues,
|
||||
values: SettingsValues,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
if previous == &values {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
let current = serde_json::to_value(previous).map_err(|_| InvalidSettings)?;
|
||||
let next = serde_json::to_value(&values).map_err(|_| InvalidSettings)?;
|
||||
let order = self.order.checked_add(1).ok_or(InvalidSettings)?;
|
||||
let mut overrides = self.overrides.as_ref().clone();
|
||||
let effective_values = self.values();
|
||||
// 每个已声明设置项整体替换,不猜测画像对象、数组或 null 的深合并含义
|
||||
for (name, value) in next.as_object().ok_or(InvalidSettings)? {
|
||||
if current.get(name) != Some(value) {
|
||||
overrides.insert(
|
||||
name.clone(),
|
||||
SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: value.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
// 只把本层实际修改的字段写回宿主层,作用域默认值不参与继承
|
||||
let values = if previous == effective_values {
|
||||
values
|
||||
} else {
|
||||
apply_overrides(self.baseline.settings(), &overrides)?
|
||||
};
|
||||
let effective = self
|
||||
.effective
|
||||
.resolve_settings(&values)
|
||||
.map_err(|_| InvalidSettings)?;
|
||||
Ok(Self {
|
||||
baseline: self.baseline.clone(),
|
||||
effective,
|
||||
overrides: Arc::new(overrides),
|
||||
execution: self.execution.clone(),
|
||||
http_timeout: self.http_timeout.clone(),
|
||||
order,
|
||||
})
|
||||
}
|
||||
|
||||
/// 持续会话的新请求读取当前宿主快照,只继承明确改写的字段
|
||||
pub fn rebase(&self, snapshot: Arc<RuntimeSnapshot>) -> Result<Self, InvalidSettings> {
|
||||
if Arc::ptr_eq(&self.baseline, &snapshot) {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
let effective = if self.overrides.is_empty() {
|
||||
snapshot.clone()
|
||||
} else {
|
||||
snapshot
|
||||
.resolve_settings(&apply_overrides(snapshot.settings(), &self.overrides)?)
|
||||
.map_err(|_| InvalidSettings)?
|
||||
};
|
||||
Ok(Self {
|
||||
baseline: snapshot,
|
||||
effective,
|
||||
overrides: self.overrides.clone(),
|
||||
execution: self.execution.clone(),
|
||||
http_timeout: self.http_timeout.clone(),
|
||||
order: self.order,
|
||||
})
|
||||
}
|
||||
|
||||
/// 认证、模型入口和插件视图共用解析规则,派生策略不改变 Key 默认值
|
||||
#[must_use]
|
||||
pub fn apply_policy(&self, policy: ClientPolicy) -> ClientPolicy {
|
||||
let values = self.resolve_execution(policy.key_id().as_str(), policy.defaults(), None);
|
||||
policy.with_settings(
|
||||
values.runtime.request_profiles(),
|
||||
values.fast_mode,
|
||||
values.client_limits,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_execution(mut self, policy: &ClientPolicy, timeout_ms: Option<u64>) -> Self {
|
||||
let input = self.resolve_execution(policy.key_id().as_str(), policy.defaults(), timeout_ms);
|
||||
let previous = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == policy.key_id().as_str());
|
||||
self.execution = Some(Arc::new(ExecutionOverrides {
|
||||
defaults: policy.defaults().clone(),
|
||||
client_key_id: policy.key_id().as_str().to_owned(),
|
||||
input: Arc::new(input),
|
||||
fast_mode: previous.and_then(|scope| scope.fast_mode.clone()),
|
||||
client_limits: previous.and_then(|scope| scope.client_limits.clone()),
|
||||
timeout_ms: previous.and_then(|scope| scope.timeout_ms.clone()),
|
||||
}));
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn execution_values(&self) -> Option<ExecutionSettings> {
|
||||
let scope = self.execution.as_ref()?;
|
||||
Some(self.resolve_execution(
|
||||
&scope.client_key_id,
|
||||
&scope.defaults,
|
||||
scope.input.timeout_ms,
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_execution(
|
||||
&self,
|
||||
key: &str,
|
||||
defaults: &ClientSettings,
|
||||
timeout_ms: Option<u64>,
|
||||
) -> ExecutionSettings {
|
||||
let overrides = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key);
|
||||
let mut runtime = self.values().clone();
|
||||
// Key 画像覆盖宿主默认,插件显式覆盖整个设置项;此优先级只在这里解释
|
||||
if !self.overrides.contains_key("request_profiles") && !defaults.request_profiles.is_empty()
|
||||
{
|
||||
let mut profiles = runtime.request_profiles().clone();
|
||||
profiles.extend(defaults.request_profiles.clone());
|
||||
runtime = runtime.with_request_profiles(profiles);
|
||||
}
|
||||
ExecutionSettings {
|
||||
runtime,
|
||||
fast_mode: overrides
|
||||
.and_then(|scope| scope.fast_mode.as_ref())
|
||||
.map_or(defaults.fast_mode, |change| change.value),
|
||||
client_limits: overrides
|
||||
.and_then(|scope| scope.client_limits.as_ref())
|
||||
.map_or(defaults.limits, |change| change.value),
|
||||
timeout_ms: overrides
|
||||
.and_then(|scope| scope.timeout_ms.as_ref())
|
||||
.map_or(timeout_ms, |change| change.value),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn execution_deadline(
|
||||
&self,
|
||||
key: &ClientApiKeyId,
|
||||
started_at: SystemTime,
|
||||
) -> Result<crate::lifecycle::Deadline, InvalidSettings> {
|
||||
let scope = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key.as_str())
|
||||
.ok_or(InvalidSettings)?;
|
||||
let timeout_ms = scope
|
||||
.timeout_ms
|
||||
.as_ref()
|
||||
.map_or(scope.input.timeout_ms, |change| change.value);
|
||||
crate::lifecycle::Deadline::from_timeout(started_at, timeout_ms.map(Duration::from_millis))
|
||||
.ok_or(InvalidSettings)
|
||||
}
|
||||
|
||||
pub fn replace_execution(
|
||||
&self,
|
||||
values: &ExecutionSettings,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
let previous = self.execution_values().ok_or(InvalidSettings)?;
|
||||
let mut updated =
|
||||
self.replace_scoped(&previous.runtime, values.runtime.clone(), instance_id)?;
|
||||
if previous.fast_mode == values.fast_mode
|
||||
&& previous.client_limits == values.client_limits
|
||||
&& previous.timeout_ms == values.timeout_ms
|
||||
{
|
||||
return Ok(updated);
|
||||
}
|
||||
let mut scope = self.execution.as_deref().cloned().ok_or(InvalidSettings)?;
|
||||
let order = if updated.order == self.order {
|
||||
self.order.checked_add(1).ok_or(InvalidSettings)?
|
||||
} else {
|
||||
updated.order
|
||||
};
|
||||
if previous.fast_mode != values.fast_mode {
|
||||
scope.fast_mode = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.fast_mode,
|
||||
});
|
||||
}
|
||||
if previous.client_limits != values.client_limits {
|
||||
scope.client_limits = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.client_limits,
|
||||
});
|
||||
}
|
||||
if previous.timeout_ms != values.timeout_ms {
|
||||
scope.timeout_ms = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.timeout_ms,
|
||||
});
|
||||
}
|
||||
updated.execution = Some(Arc::new(scope));
|
||||
updated.order = order;
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn execution_timeout(&self, key: &ClientApiKeyId) -> Option<Duration> {
|
||||
self.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key.as_str())
|
||||
.and_then(|scope| scope.timeout_ms.as_ref())
|
||||
.and_then(|change| change.value.map(Duration::from_millis))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_http_timeout(mut self, timeout_ms: Option<u64>) -> Self {
|
||||
self.http_timeout.get_or_insert(HttpTimeout {
|
||||
input_ms: timeout_ms,
|
||||
change: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
pub fn replace_http_timeout(
|
||||
mut self,
|
||||
timeout_ms: Option<u64>,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
let timeout = self.http_timeout.get_or_insert(HttpTimeout {
|
||||
input_ms: None,
|
||||
change: None,
|
||||
});
|
||||
if timeout
|
||||
.change
|
||||
.as_ref()
|
||||
.map_or(timeout.input_ms, |change| change.value)
|
||||
!= timeout_ms
|
||||
{
|
||||
self.order = self.order.checked_add(1).ok_or(InvalidSettings)?;
|
||||
timeout.change = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order: self.order,
|
||||
value: timeout_ms,
|
||||
});
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_overrides(
|
||||
values: &SettingsValues,
|
||||
overrides: &BTreeMap<String, SettingOverride>,
|
||||
) -> Result<SettingsValues, InvalidSettings> {
|
||||
let mut values = serde_json::to_value(values).map_err(|_| InvalidSettings)?;
|
||||
let object = values.as_object_mut().ok_or(InvalidSettings)?;
|
||||
for (name, change) in overrides {
|
||||
object.insert(name.clone(), change.value.clone());
|
||||
}
|
||||
serde_json::from_value(values).map_err(|_| InvalidSettings)
|
||||
}
|
||||
@@ -220,7 +220,7 @@ pub enum RuntimeSnapshotCompileError {
|
||||
pub struct RuntimeSnapshotCompiler {
|
||||
store: Arc<dyn SnapshotStorePort>,
|
||||
catalogs: Arc<dyn ProviderCatalogPort>,
|
||||
extensions: Option<Arc<dyn crate::runtime::extensions::ExtensionPreparationPort>>,
|
||||
extensions: Option<Arc<dyn crate::routing::extensions::ExtensionPreparationPort>>,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotCompiler {
|
||||
@@ -239,7 +239,7 @@ impl RuntimeSnapshotCompiler {
|
||||
#[must_use]
|
||||
pub fn with_extensions(
|
||||
mut self,
|
||||
extensions: Arc<dyn crate::runtime::extensions::ExtensionPreparationPort>,
|
||||
extensions: Arc<dyn crate::routing::extensions::ExtensionPreparationPort>,
|
||||
) -> Self {
|
||||
self.extensions = Some(extensions);
|
||||
self
|
||||
@@ -275,7 +275,7 @@ impl RuntimeSnapshotCompiler {
|
||||
// 同一配置 revision 的重试复用扩展候选,保留插件策略的发布一致性
|
||||
let mut prepared_extensions: Option<(
|
||||
ConfigRevision,
|
||||
crate::runtime::extensions::ExtensionSetReference,
|
||||
crate::routing::extensions::ExtensionSetReference,
|
||||
)> = None;
|
||||
for _ in 0..MAXIMUM_CATALOG_STABILITY_ATTEMPTS {
|
||||
let facts = self
|
||||
@@ -524,7 +524,7 @@ async fn compile_runtime_snapshot(
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RuntimeSnapshot {
|
||||
settings: Arc<CompiledSettings>,
|
||||
extensions: Option<crate::runtime::extensions::ExtensionSetReference>,
|
||||
extensions: Option<crate::routing::extensions::ExtensionSetReference>,
|
||||
revision: ConfigRevision,
|
||||
providers: Arc<BTreeSet<ProviderKind>>,
|
||||
provider_models: Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>>,
|
||||
@@ -570,14 +570,14 @@ impl RuntimeSnapshot {
|
||||
#[must_use]
|
||||
pub fn with_extensions(
|
||||
mut self,
|
||||
extensions: Option<crate::runtime::extensions::ExtensionSetReference>,
|
||||
extensions: Option<crate::routing::extensions::ExtensionSetReference>,
|
||||
) -> Self {
|
||||
self.extensions = extensions;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn extensions(&self) -> Option<&crate::runtime::extensions::ExtensionSetReference> {
|
||||
pub const fn extensions(&self) -> Option<&crate::routing::extensions::ExtensionSetReference> {
|
||||
self.extensions.as_ref()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,542 +1,10 @@
|
||||
//! 运行时快照的原子发布、健康状态与跨进程版本收敛
|
||||
|
||||
pub mod extensions;
|
||||
mod publications;
|
||||
mod publisher;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex as SyncMutex, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use futures::lock::Mutex;
|
||||
use futures::{FutureExt as _, Stream, StreamExt as _, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
|
||||
use crate::health::{HealthProbe, HealthState};
|
||||
use crate::identity::ProviderKind;
|
||||
use crate::lifecycle::CancellationToken;
|
||||
use crate::routing::snapshot::{
|
||||
RuntimeSnapshot, RuntimeSnapshotCompileError, RuntimeSnapshotCompiler,
|
||||
pub use publications::{RuntimeSnapshotHandle, RuntimeSnapshotUnavailable};
|
||||
pub use publisher::{
|
||||
RuntimeSnapshotPublisher, SnapshotControl, SnapshotRevisionStream, SnapshotSubscriptionError,
|
||||
SnapshotSubscriptionPort, runtime_revision_needs_refresh,
|
||||
};
|
||||
use crate::routing::{ConfigRevision, ProviderCatalogGeneration};
|
||||
use crate::task::{
|
||||
DaemonRestartPolicy, DaemonTask, ScheduledTask, WorkerContribution, WorkerCycleContext,
|
||||
WorkerDefinitionError, WorkerId, WorkerKind, WorkerRegistration, WorkerRunnable,
|
||||
WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
|
||||
const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const INITIAL_BACKOFF: Duration = Duration::from_secs(1);
|
||||
const MAXIMUM_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const UNUSED_LEASE_TTL: Duration = Duration::from_secs(30);
|
||||
const UNUSED_LEASE_RENEWAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// 不泄漏订阅基础设施细节的通知错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("runtime snapshot notification is unavailable")]
|
||||
pub struct SnapshotSubscriptionError;
|
||||
|
||||
impl SnapshotSubscriptionError {
|
||||
#[must_use]
|
||||
pub const fn unavailable() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的配置 revision 通知流;权威 revision 始终由 Store 端口读取
|
||||
pub type SnapshotRevisionStream =
|
||||
Pin<Box<dyn Stream<Item = Result<ConfigRevision, SnapshotSubscriptionError>> + Send + 'static>>;
|
||||
|
||||
/// 跨进程 revision 通知的基础设施中立端口
|
||||
pub trait SnapshotSubscriptionPort: Send + Sync {
|
||||
fn publish_snapshot_revision(
|
||||
&self,
|
||||
revision: ConfigRevision,
|
||||
) -> BoxFuture<'_, Result<(), SnapshotSubscriptionError>>;
|
||||
|
||||
fn subscribe_snapshot_revisions(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<SnapshotRevisionStream, SnapshotSubscriptionError>>;
|
||||
}
|
||||
|
||||
/// 请求级冻结失败;此状态必须 fail closed
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("runtime snapshot is unavailable")]
|
||||
pub struct RuntimeSnapshotUnavailable;
|
||||
|
||||
/// RuntimeSnapshot 原子发布和请求级冻结句柄
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RuntimeSnapshotHandle {
|
||||
current: Arc<RwLock<Option<Arc<RuntimeSnapshot>>>>,
|
||||
publications: Arc<SyncMutex<Vec<futures::channel::mpsc::Sender<()>>>>,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotHandle {
|
||||
#[must_use]
|
||||
pub fn new(initial: RuntimeSnapshot) -> Self {
|
||||
Self {
|
||||
current: Arc::new(RwLock::new(Some(Arc::new(initial)))),
|
||||
publications: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn publish(&self, snapshot: RuntimeSnapshot) {
|
||||
*write_unpoisoned(&self.current) = Some(Arc::new(snapshot));
|
||||
self.notify_publication();
|
||||
}
|
||||
|
||||
pub fn suspend(&self) {
|
||||
*write_unpoisoned(&self.current) = None;
|
||||
self.notify_publication();
|
||||
}
|
||||
|
||||
/// 合并发布与暂停通知;订阅者收到后读取当前快照,不依赖逐条投递
|
||||
pub fn subscribe_publications(&self) -> futures::channel::mpsc::Receiver<()> {
|
||||
let (sender, receiver) = futures::channel::mpsc::channel(0);
|
||||
self.publications
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(sender);
|
||||
receiver
|
||||
}
|
||||
|
||||
fn notify_publication(&self) {
|
||||
self.publications
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.retain_mut(|sender| match sender.try_send(()) {
|
||||
Ok(()) => true,
|
||||
Err(error) => error.is_full(),
|
||||
});
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn revision(&self) -> Option<ConfigRevision> {
|
||||
read_unpoisoned(&self.current)
|
||||
.as_ref()
|
||||
.map(|snapshot| snapshot.revision())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn provider_catalog_generations(
|
||||
&self,
|
||||
) -> Option<BTreeMap<ProviderKind, ProviderCatalogGeneration>> {
|
||||
read_unpoisoned(&self.current)
|
||||
.as_ref()
|
||||
.map(|snapshot| snapshot.provider_catalog_generations().clone())
|
||||
}
|
||||
|
||||
/// 控制面只读诊断可观察尚未就绪的发布候选;数据面仍必须通过 [`Self::acquire`]
|
||||
#[must_use]
|
||||
pub fn snapshot_for_diagnostics(&self) -> Option<Arc<RuntimeSnapshot>> {
|
||||
read_unpoisoned(&self.current).clone()
|
||||
}
|
||||
|
||||
/// 冻结当前 Arc;后续发布不改变已经开始的请求
|
||||
pub fn acquire(&self) -> Result<Arc<RuntimeSnapshot>, RuntimeSnapshotUnavailable> {
|
||||
read_unpoisoned(&self.current)
|
||||
.clone()
|
||||
.filter(|snapshot| snapshot.extensions().is_none_or(|set| set.can_serve()))
|
||||
.ok_or(RuntimeSnapshotUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthProbe for RuntimeSnapshotHandle {
|
||||
fn name(&self) -> &'static str {
|
||||
"runtime_snapshot"
|
||||
}
|
||||
|
||||
fn check(&self) -> BoxFuture<'_, HealthState> {
|
||||
Box::pin(async move {
|
||||
if self.acquire().is_ok() {
|
||||
HealthState::Healthy
|
||||
} else {
|
||||
HealthState::Unhealthy("Runtime snapshot is unavailable".to_owned())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Admin 提交配置后触发本进程刷新与跨进程通知的对象安全端口
|
||||
pub trait SnapshotControl: Send + Sync {
|
||||
fn publish_committed(&self, committed_revision: ConfigRevision) -> BoxFuture<'_, ()>;
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RefreshPriorityState {
|
||||
pending_commits: usize,
|
||||
active_background: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
struct CommittedRefreshGuard {
|
||||
priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
}
|
||||
|
||||
impl CommittedRefreshGuard {
|
||||
fn begin(priority: Arc<SyncMutex<RefreshPriorityState>>) -> Self {
|
||||
let active = {
|
||||
let mut state = lock_unpoisoned(&priority);
|
||||
state.pending_commits += 1;
|
||||
state.active_background.clone()
|
||||
};
|
||||
// 先登记待发布提交,再取消正在等待目录的后台刷新,避免后者抢先重入
|
||||
if let Some(active) = active {
|
||||
active.cancel();
|
||||
}
|
||||
Self { priority }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CommittedRefreshGuard {
|
||||
fn drop(&mut self) {
|
||||
lock_unpoisoned(&self.priority).pending_commits -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundRefreshGuard {
|
||||
priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
|
||||
impl BackgroundRefreshGuard {
|
||||
fn begin(priority: Arc<SyncMutex<RefreshPriorityState>>) -> Option<Self> {
|
||||
let mut state = lock_unpoisoned(&priority);
|
||||
if state.pending_commits != 0 {
|
||||
return None;
|
||||
}
|
||||
let cancellation = CancellationToken::new();
|
||||
state.active_background = Some(cancellation.clone());
|
||||
drop(state);
|
||||
Some(Self {
|
||||
priority,
|
||||
cancellation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundRefreshGuard {
|
||||
fn drop(&mut self) {
|
||||
lock_unpoisoned(&self.priority).active_background = None;
|
||||
}
|
||||
}
|
||||
|
||||
// 仅中断未完成的后台读取或编译;配置提交负责发布最新事实并发送通知
|
||||
async fn preempt_for_committed_refresh<T>(
|
||||
cancellation: Option<&CancellationToken>,
|
||||
operation: impl Future<Output = T>,
|
||||
) -> Result<T, RuntimeSnapshotCompileError> {
|
||||
let Some(cancellation) = cancellation else {
|
||||
return Ok(operation.await);
|
||||
};
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let operation = operation.fuse();
|
||||
pin_mut!(cancelled, operation);
|
||||
select_biased! {
|
||||
_ = cancelled => Err(RuntimeSnapshotCompileError::RevisionChanged),
|
||||
result = operation => Ok(result),
|
||||
}
|
||||
}
|
||||
|
||||
/// 配置提交后的本进程快照发布与跨进程失效通知
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeSnapshotPublisher {
|
||||
compiler: Arc<RuntimeSnapshotCompiler>,
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
refresh_lock: Arc<Mutex<()>>,
|
||||
refresh_priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
}
|
||||
|
||||
enum RefreshMode {
|
||||
Required,
|
||||
Reconcile,
|
||||
Committed,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotPublisher {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
compiler: Arc<RuntimeSnapshotCompiler>,
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
compiler,
|
||||
snapshots,
|
||||
subscriptions,
|
||||
refresh_lock: Arc::new(Mutex::new(())),
|
||||
refresh_priority: Arc::new(SyncMutex::new(RefreshPriorityState::default())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 串行重编译并替换本进程快照;无法确认配置时暂停新请求
|
||||
pub async fn refresh(&self) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
self.refresh_with_mode(RefreshMode::Required).await
|
||||
}
|
||||
|
||||
async fn refresh_with_mode(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
// 提交先登记并中断慢后台读取;发布/暂停仍在完整临界区内串行执行
|
||||
let _committed = matches!(mode, RefreshMode::Committed)
|
||||
.then(|| CommittedRefreshGuard::begin(Arc::clone(&self.refresh_priority)));
|
||||
let _refresh = self.refresh_lock.lock().await;
|
||||
self.refresh_locked_with_priority(mode).await
|
||||
}
|
||||
|
||||
async fn refresh_locked_with_priority(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
if matches!(mode, RefreshMode::Committed) {
|
||||
return self.refresh_locked(mode, None).await;
|
||||
}
|
||||
let background = BackgroundRefreshGuard::begin(Arc::clone(&self.refresh_priority))
|
||||
.ok_or(RuntimeSnapshotCompileError::RevisionChanged)?;
|
||||
self.refresh_locked(mode, Some(&background.cancellation))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn refresh_locked(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
let configuration_changed = if matches!(mode, RefreshMode::Reconcile) {
|
||||
let persisted_revision = preempt_for_committed_refresh(
|
||||
cancellation,
|
||||
self.compiler.store().current_config_revision(),
|
||||
)
|
||||
.await?
|
||||
.map_err(|_| {
|
||||
self.snapshots.suspend();
|
||||
RuntimeSnapshotCompileError::StoreUnavailable
|
||||
})?;
|
||||
let changed = runtime_revision_needs_refresh(
|
||||
self.published_revision().map(ConfigRevision::get),
|
||||
persisted_revision.get(),
|
||||
);
|
||||
if !changed
|
||||
&& !self.provider_catalogs_need_refresh()
|
||||
&& self
|
||||
.snapshots
|
||||
.acquire()
|
||||
.is_ok_and(|snapshot| snapshot.extensions().is_none_or(|set| set.is_ready()))
|
||||
{
|
||||
return Ok(persisted_revision);
|
||||
}
|
||||
changed
|
||||
} else {
|
||||
true
|
||||
};
|
||||
let compiled = if matches!(mode, RefreshMode::Committed) {
|
||||
match self.snapshots.snapshot_for_diagnostics() {
|
||||
Some(previous) => self.compiler.compile_with_cached_catalog(&previous).await,
|
||||
None => self.compiler.compile().await,
|
||||
}
|
||||
} else {
|
||||
preempt_for_committed_refresh(cancellation, self.compiler.compile()).await?
|
||||
};
|
||||
let snapshot = match compiled {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
// 目录对账失败可继续服务旧快照;配置或权限变化无法确认时须暂停
|
||||
if configuration_changed {
|
||||
self.snapshots.suspend();
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let revision = snapshot.revision();
|
||||
self.snapshots.publish(snapshot);
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn published_revision(&self) -> Option<ConfigRevision> {
|
||||
self.snapshots.revision()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn provider_catalogs_need_refresh(&self) -> bool {
|
||||
let Ok(snapshot) = self.snapshots.acquire() else {
|
||||
return true;
|
||||
};
|
||||
self.compiler.provider_catalog_generations() != *snapshot.provider_catalog_generations()
|
||||
}
|
||||
|
||||
/// 数据库提交不能被目录或通知基础设施的暂时故障伪装成回滚
|
||||
async fn publish_committed_inner(&self, committed_revision: ConfigRevision) {
|
||||
if let Err(error) = self.refresh_with_mode(RefreshMode::Committed).await {
|
||||
tracing::warn!(?error, "committed runtime snapshot refresh failed");
|
||||
}
|
||||
self.notify_committed_revision(committed_revision).await;
|
||||
}
|
||||
|
||||
async fn notify_committed_revision(&self, committed_revision: ConfigRevision) {
|
||||
if let Err(error) = self
|
||||
.subscriptions
|
||||
.publish_snapshot_revision(committed_revision)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
?error,
|
||||
revision = committed_revision.get(),
|
||||
"runtime snapshot revision notification failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// 交给 Host 的周期对账与长驻订阅任务
|
||||
pub fn worker_contributions(&self) -> Result<Vec<WorkerContribution>, WorkerDefinitionError> {
|
||||
let reconciliation_id = WorkerId::try_new(
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
"runtime_snapshot",
|
||||
)?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
RECONCILIATION_INTERVAL,
|
||||
INITIAL_BACKOFF,
|
||||
MAXIMUM_BACKOFF,
|
||||
UNUSED_LEASE_TTL,
|
||||
UNUSED_LEASE_RENEWAL,
|
||||
)?;
|
||||
let reconciliation = WorkerRegistration::try_new(
|
||||
reconciliation_id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: None,
|
||||
task: Box::new(RuntimeSnapshotReconciliationTask {
|
||||
publisher: self.clone(),
|
||||
}),
|
||||
},
|
||||
)?;
|
||||
let subscription_id =
|
||||
WorkerId::try_new(WorkerKind::RuntimeChangeSubscription, "runtime_snapshot")?;
|
||||
let restart = DaemonRestartPolicy::try_new(INITIAL_BACKOFF, MAXIMUM_BACKOFF)?;
|
||||
let subscription = WorkerRegistration::try_new(
|
||||
subscription_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(RuntimeSnapshotSubscriptionTask {
|
||||
subscriptions: Arc::clone(&self.subscriptions),
|
||||
publisher: self.clone(),
|
||||
}),
|
||||
},
|
||||
)?;
|
||||
Ok(vec![
|
||||
WorkerContribution::Registration(reconciliation),
|
||||
WorkerContribution::Registration(subscription),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
impl SnapshotControl for RuntimeSnapshotPublisher {
|
||||
fn publish_committed(&self, committed_revision: ConfigRevision) -> BoxFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.publish_committed_inner(committed_revision).await;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RuntimeSnapshotReconciliationTask {
|
||||
publisher: RuntimeSnapshotPublisher,
|
||||
}
|
||||
|
||||
impl ScheduledTask for RuntimeSnapshotReconciliationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
_context: WorkerCycleContext,
|
||||
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
self.publisher
|
||||
.refresh_with_mode(RefreshMode::Reconcile)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|source| {
|
||||
WorkerTaskError::safe("runtime snapshot reconciliation failed")
|
||||
.with_source(source)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RuntimeSnapshotSubscriptionTask {
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
publisher: RuntimeSnapshotPublisher,
|
||||
}
|
||||
|
||||
impl DaemonTask for RuntimeSnapshotSubscriptionTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut retry_delay = INITIAL_BACKOFF;
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let subscription = self.subscriptions.subscribe_snapshot_revisions().await;
|
||||
let mut subscription = match subscription {
|
||||
Ok(subscription) => {
|
||||
retry_delay = INITIAL_BACKOFF;
|
||||
subscription
|
||||
}
|
||||
Err(_) => {
|
||||
wait_or_cancel(&cancellation, retry_delay).await;
|
||||
retry_delay = (retry_delay * 2).min(MAXIMUM_BACKOFF);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let next = subscription.next().fuse();
|
||||
pin_mut!(cancelled, next);
|
||||
let notified = select_biased! {
|
||||
_ = cancelled => return Ok(()),
|
||||
next = next => next,
|
||||
};
|
||||
match notified {
|
||||
Some(Ok(_)) => {
|
||||
let _ = self.publisher.refresh().await;
|
||||
}
|
||||
Some(Err(_)) | None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_or_cancel(cancellation: &CancellationToken, duration: Duration) {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let delay = Delay::new(duration).fuse();
|
||||
pin_mut!(cancelled, delay);
|
||||
select_biased! {
|
||||
_ = cancelled => {},
|
||||
_ = delay => {},
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前发布版本与持久版本不一致时必须重载;缺失和回退同样 fail closed
|
||||
#[must_use]
|
||||
pub fn runtime_revision_needs_refresh(
|
||||
published_revision: Option<u64>,
|
||||
persisted_revision: u64,
|
||||
) -> bool {
|
||||
published_revision != Some(persisted_revision)
|
||||
}
|
||||
|
||||
fn read_unpoisoned<T>(lock: &RwLock<T>) -> std::sync::RwLockReadGuard<'_, T> {
|
||||
lock.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn write_unpoisoned<T>(lock: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
|
||||
lock.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn lock_unpoisoned<T>(lock: &SyncMutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
lock.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! 当前路由快照的原子发布、冻结与健康投影
|
||||
|
||||
use crate::{
|
||||
health::{HealthProbe, HealthState},
|
||||
identity::ProviderKind,
|
||||
routing::{ConfigRevision, ProviderCatalogGeneration, RuntimeSnapshot},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex as SyncMutex, RwLock},
|
||||
};
|
||||
|
||||
/// 请求级冻结失败;此状态必须 fail closed
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("runtime snapshot is unavailable")]
|
||||
pub struct RuntimeSnapshotUnavailable;
|
||||
|
||||
/// RuntimeSnapshot 原子发布和请求级冻结句柄
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RuntimeSnapshotHandle {
|
||||
current: Arc<RwLock<Option<Arc<RuntimeSnapshot>>>>,
|
||||
publications: Arc<SyncMutex<Vec<futures::channel::mpsc::Sender<()>>>>,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotHandle {
|
||||
#[must_use]
|
||||
pub fn new(initial: RuntimeSnapshot) -> Self {
|
||||
Self {
|
||||
current: Arc::new(RwLock::new(Some(Arc::new(initial)))),
|
||||
publications: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn publish(&self, snapshot: RuntimeSnapshot) {
|
||||
*write_unpoisoned(&self.current) = Some(Arc::new(snapshot));
|
||||
self.notify_publication();
|
||||
}
|
||||
|
||||
pub fn suspend(&self) {
|
||||
*write_unpoisoned(&self.current) = None;
|
||||
self.notify_publication();
|
||||
}
|
||||
|
||||
/// 合并发布与暂停通知;订阅者收到后读取当前快照,不依赖逐条投递
|
||||
pub fn subscribe_publications(&self) -> futures::channel::mpsc::Receiver<()> {
|
||||
let (sender, receiver) = futures::channel::mpsc::channel(0);
|
||||
self.publications
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(sender);
|
||||
receiver
|
||||
}
|
||||
|
||||
fn notify_publication(&self) {
|
||||
self.publications
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.retain_mut(|sender| match sender.try_send(()) {
|
||||
Ok(()) => true,
|
||||
Err(error) => error.is_full(),
|
||||
});
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn revision(&self) -> Option<ConfigRevision> {
|
||||
read_unpoisoned(&self.current)
|
||||
.as_ref()
|
||||
.map(|snapshot| snapshot.revision())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn provider_catalog_generations(
|
||||
&self,
|
||||
) -> Option<BTreeMap<ProviderKind, ProviderCatalogGeneration>> {
|
||||
read_unpoisoned(&self.current)
|
||||
.as_ref()
|
||||
.map(|snapshot| snapshot.provider_catalog_generations().clone())
|
||||
}
|
||||
|
||||
/// 控制面只读诊断可观察尚未就绪的发布候选;数据面仍必须通过 [`Self::acquire`]
|
||||
#[must_use]
|
||||
pub fn snapshot_for_diagnostics(&self) -> Option<Arc<RuntimeSnapshot>> {
|
||||
read_unpoisoned(&self.current).clone()
|
||||
}
|
||||
|
||||
/// 冻结当前 Arc;后续发布不改变已经开始的请求
|
||||
pub fn acquire(&self) -> Result<Arc<RuntimeSnapshot>, RuntimeSnapshotUnavailable> {
|
||||
read_unpoisoned(&self.current)
|
||||
.clone()
|
||||
.filter(|snapshot| snapshot.extensions().is_none_or(|set| set.can_serve()))
|
||||
.ok_or(RuntimeSnapshotUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthProbe for RuntimeSnapshotHandle {
|
||||
fn name(&self) -> &'static str {
|
||||
"runtime_snapshot"
|
||||
}
|
||||
|
||||
fn check(&self) -> BoxFuture<'_, HealthState> {
|
||||
Box::pin(async move {
|
||||
if self.acquire().is_ok() {
|
||||
HealthState::Healthy
|
||||
} else {
|
||||
HealthState::Unhealthy("Runtime snapshot is unavailable".to_owned())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn read_unpoisoned<T>(lock: &RwLock<T>) -> std::sync::RwLockReadGuard<'_, T> {
|
||||
lock.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn write_unpoisoned<T>(lock: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
|
||||
lock.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
//! 运行时快照的原子发布、健康状态与跨进程版本收敛
|
||||
|
||||
use super::publications::RuntimeSnapshotHandle;
|
||||
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex as SyncMutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use futures::lock::Mutex;
|
||||
use futures::{FutureExt as _, Stream, StreamExt as _, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
|
||||
use crate::lifecycle::CancellationToken;
|
||||
use crate::routing::ConfigRevision;
|
||||
use crate::routing::snapshot::{RuntimeSnapshotCompileError, RuntimeSnapshotCompiler};
|
||||
use crate::task::{
|
||||
DaemonRestartPolicy, DaemonTask, ScheduledTask, WorkerContribution, WorkerCycleContext,
|
||||
WorkerDefinitionError, WorkerId, WorkerKind, WorkerRegistration, WorkerRunnable,
|
||||
WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
|
||||
const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const INITIAL_BACKOFF: Duration = Duration::from_secs(1);
|
||||
const MAXIMUM_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const UNUSED_LEASE_TTL: Duration = Duration::from_secs(30);
|
||||
const UNUSED_LEASE_RENEWAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// 不泄漏订阅基础设施细节的通知错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("runtime snapshot notification is unavailable")]
|
||||
pub struct SnapshotSubscriptionError;
|
||||
|
||||
impl SnapshotSubscriptionError {
|
||||
#[must_use]
|
||||
pub const fn unavailable() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的配置 revision 通知流;权威 revision 始终由 Store 端口读取
|
||||
pub type SnapshotRevisionStream =
|
||||
Pin<Box<dyn Stream<Item = Result<ConfigRevision, SnapshotSubscriptionError>> + Send + 'static>>;
|
||||
|
||||
/// 跨进程 revision 通知的基础设施中立端口
|
||||
pub trait SnapshotSubscriptionPort: Send + Sync {
|
||||
fn publish_snapshot_revision(
|
||||
&self,
|
||||
revision: ConfigRevision,
|
||||
) -> BoxFuture<'_, Result<(), SnapshotSubscriptionError>>;
|
||||
|
||||
fn subscribe_snapshot_revisions(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<SnapshotRevisionStream, SnapshotSubscriptionError>>;
|
||||
}
|
||||
|
||||
/// Admin 提交配置后触发本进程刷新与跨进程通知的对象安全端口
|
||||
pub trait SnapshotControl: Send + Sync {
|
||||
fn publish_committed(&self, committed_revision: ConfigRevision) -> BoxFuture<'_, ()>;
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RefreshPriorityState {
|
||||
pending_commits: usize,
|
||||
active_background: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
struct CommittedRefreshGuard {
|
||||
priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
}
|
||||
|
||||
impl CommittedRefreshGuard {
|
||||
fn begin(priority: Arc<SyncMutex<RefreshPriorityState>>) -> Self {
|
||||
let active = {
|
||||
let mut state = lock_unpoisoned(&priority);
|
||||
state.pending_commits += 1;
|
||||
state.active_background.clone()
|
||||
};
|
||||
// 先登记待发布提交,再取消正在等待目录的后台刷新,避免后者抢先重入
|
||||
if let Some(active) = active {
|
||||
active.cancel();
|
||||
}
|
||||
Self { priority }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CommittedRefreshGuard {
|
||||
fn drop(&mut self) {
|
||||
lock_unpoisoned(&self.priority).pending_commits -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundRefreshGuard {
|
||||
priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
|
||||
impl BackgroundRefreshGuard {
|
||||
fn begin(priority: Arc<SyncMutex<RefreshPriorityState>>) -> Option<Self> {
|
||||
let mut state = lock_unpoisoned(&priority);
|
||||
if state.pending_commits != 0 {
|
||||
return None;
|
||||
}
|
||||
let cancellation = CancellationToken::new();
|
||||
state.active_background = Some(cancellation.clone());
|
||||
drop(state);
|
||||
Some(Self {
|
||||
priority,
|
||||
cancellation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundRefreshGuard {
|
||||
fn drop(&mut self) {
|
||||
lock_unpoisoned(&self.priority).active_background = None;
|
||||
}
|
||||
}
|
||||
|
||||
// 仅中断未完成的后台读取或编译;配置提交负责发布最新事实并发送通知
|
||||
async fn preempt_for_committed_refresh<T>(
|
||||
cancellation: Option<&CancellationToken>,
|
||||
operation: impl Future<Output = T>,
|
||||
) -> Result<T, RuntimeSnapshotCompileError> {
|
||||
let Some(cancellation) = cancellation else {
|
||||
return Ok(operation.await);
|
||||
};
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let operation = operation.fuse();
|
||||
pin_mut!(cancelled, operation);
|
||||
select_biased! {
|
||||
_ = cancelled => Err(RuntimeSnapshotCompileError::RevisionChanged),
|
||||
result = operation => Ok(result),
|
||||
}
|
||||
}
|
||||
|
||||
/// 配置提交后的本进程快照发布与跨进程失效通知
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeSnapshotPublisher {
|
||||
compiler: Arc<RuntimeSnapshotCompiler>,
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
refresh_lock: Arc<Mutex<()>>,
|
||||
refresh_priority: Arc<SyncMutex<RefreshPriorityState>>,
|
||||
}
|
||||
|
||||
enum RefreshMode {
|
||||
Required,
|
||||
Reconcile,
|
||||
Committed,
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotPublisher {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
compiler: Arc<RuntimeSnapshotCompiler>,
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
compiler,
|
||||
snapshots,
|
||||
subscriptions,
|
||||
refresh_lock: Arc::new(Mutex::new(())),
|
||||
refresh_priority: Arc::new(SyncMutex::new(RefreshPriorityState::default())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 串行重编译并替换本进程快照;无法确认配置时暂停新请求
|
||||
pub async fn refresh(&self) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
self.refresh_with_mode(RefreshMode::Required).await
|
||||
}
|
||||
|
||||
async fn refresh_with_mode(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
// 提交先登记并中断慢后台读取;发布/暂停仍在完整临界区内串行执行
|
||||
let _committed = matches!(mode, RefreshMode::Committed)
|
||||
.then(|| CommittedRefreshGuard::begin(Arc::clone(&self.refresh_priority)));
|
||||
let _refresh = self.refresh_lock.lock().await;
|
||||
self.refresh_locked_with_priority(mode).await
|
||||
}
|
||||
|
||||
async fn refresh_locked_with_priority(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
if matches!(mode, RefreshMode::Committed) {
|
||||
return self.refresh_locked(mode, None).await;
|
||||
}
|
||||
let background = BackgroundRefreshGuard::begin(Arc::clone(&self.refresh_priority))
|
||||
.ok_or(RuntimeSnapshotCompileError::RevisionChanged)?;
|
||||
self.refresh_locked(mode, Some(&background.cancellation))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn refresh_locked(
|
||||
&self,
|
||||
mode: RefreshMode,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<ConfigRevision, RuntimeSnapshotCompileError> {
|
||||
let configuration_changed = if matches!(mode, RefreshMode::Reconcile) {
|
||||
let persisted_revision = preempt_for_committed_refresh(
|
||||
cancellation,
|
||||
self.compiler.store().current_config_revision(),
|
||||
)
|
||||
.await?
|
||||
.map_err(|_| {
|
||||
self.snapshots.suspend();
|
||||
RuntimeSnapshotCompileError::StoreUnavailable
|
||||
})?;
|
||||
let changed = runtime_revision_needs_refresh(
|
||||
self.published_revision().map(ConfigRevision::get),
|
||||
persisted_revision.get(),
|
||||
);
|
||||
if !changed
|
||||
&& !self.provider_catalogs_need_refresh()
|
||||
&& self
|
||||
.snapshots
|
||||
.acquire()
|
||||
.is_ok_and(|snapshot| snapshot.extensions().is_none_or(|set| set.is_ready()))
|
||||
{
|
||||
return Ok(persisted_revision);
|
||||
}
|
||||
changed
|
||||
} else {
|
||||
true
|
||||
};
|
||||
let compiled = if matches!(mode, RefreshMode::Committed) {
|
||||
match self.snapshots.snapshot_for_diagnostics() {
|
||||
Some(previous) => self.compiler.compile_with_cached_catalog(&previous).await,
|
||||
None => self.compiler.compile().await,
|
||||
}
|
||||
} else {
|
||||
preempt_for_committed_refresh(cancellation, self.compiler.compile()).await?
|
||||
};
|
||||
let snapshot = match compiled {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
// 目录对账失败可继续服务旧快照;配置或权限变化无法确认时须暂停
|
||||
if configuration_changed {
|
||||
self.snapshots.suspend();
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let revision = snapshot.revision();
|
||||
self.snapshots.publish(snapshot);
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn published_revision(&self) -> Option<ConfigRevision> {
|
||||
self.snapshots.revision()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn provider_catalogs_need_refresh(&self) -> bool {
|
||||
let Ok(snapshot) = self.snapshots.acquire() else {
|
||||
return true;
|
||||
};
|
||||
self.compiler.provider_catalog_generations() != *snapshot.provider_catalog_generations()
|
||||
}
|
||||
|
||||
/// 数据库提交不能被目录或通知基础设施的暂时故障伪装成回滚
|
||||
async fn publish_committed_inner(&self, committed_revision: ConfigRevision) {
|
||||
if let Err(error) = self.refresh_with_mode(RefreshMode::Committed).await {
|
||||
tracing::warn!(?error, "committed runtime snapshot refresh failed");
|
||||
}
|
||||
self.notify_committed_revision(committed_revision).await;
|
||||
}
|
||||
|
||||
async fn notify_committed_revision(&self, committed_revision: ConfigRevision) {
|
||||
if let Err(error) = self
|
||||
.subscriptions
|
||||
.publish_snapshot_revision(committed_revision)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
?error,
|
||||
revision = committed_revision.get(),
|
||||
"runtime snapshot revision notification failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// 交给 Host 的周期对账与长驻订阅任务
|
||||
pub fn worker_contributions(&self) -> Result<Vec<WorkerContribution>, WorkerDefinitionError> {
|
||||
let reconciliation_id = WorkerId::try_new(
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
"runtime_snapshot",
|
||||
)?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
RECONCILIATION_INTERVAL,
|
||||
INITIAL_BACKOFF,
|
||||
MAXIMUM_BACKOFF,
|
||||
UNUSED_LEASE_TTL,
|
||||
UNUSED_LEASE_RENEWAL,
|
||||
)?;
|
||||
let reconciliation = WorkerRegistration::try_new(
|
||||
reconciliation_id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: None,
|
||||
task: Box::new(RuntimeSnapshotReconciliationTask {
|
||||
publisher: self.clone(),
|
||||
}),
|
||||
},
|
||||
)?;
|
||||
let subscription_id =
|
||||
WorkerId::try_new(WorkerKind::RuntimeChangeSubscription, "runtime_snapshot")?;
|
||||
let restart = DaemonRestartPolicy::try_new(INITIAL_BACKOFF, MAXIMUM_BACKOFF)?;
|
||||
let subscription = WorkerRegistration::try_new(
|
||||
subscription_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(RuntimeSnapshotSubscriptionTask {
|
||||
subscriptions: Arc::clone(&self.subscriptions),
|
||||
publisher: self.clone(),
|
||||
}),
|
||||
},
|
||||
)?;
|
||||
Ok(vec![
|
||||
WorkerContribution::Registration(reconciliation),
|
||||
WorkerContribution::Registration(subscription),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
impl SnapshotControl for RuntimeSnapshotPublisher {
|
||||
fn publish_committed(&self, committed_revision: ConfigRevision) -> BoxFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.publish_committed_inner(committed_revision).await;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RuntimeSnapshotReconciliationTask {
|
||||
publisher: RuntimeSnapshotPublisher,
|
||||
}
|
||||
|
||||
impl ScheduledTask for RuntimeSnapshotReconciliationTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
_context: WorkerCycleContext,
|
||||
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
self.publisher
|
||||
.refresh_with_mode(RefreshMode::Reconcile)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|source| {
|
||||
WorkerTaskError::safe("runtime snapshot reconciliation failed")
|
||||
.with_source(source)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RuntimeSnapshotSubscriptionTask {
|
||||
subscriptions: Arc<dyn SnapshotSubscriptionPort>,
|
||||
publisher: RuntimeSnapshotPublisher,
|
||||
}
|
||||
|
||||
impl DaemonTask for RuntimeSnapshotSubscriptionTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut retry_delay = INITIAL_BACKOFF;
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let subscription = self.subscriptions.subscribe_snapshot_revisions().await;
|
||||
let mut subscription = match subscription {
|
||||
Ok(subscription) => {
|
||||
retry_delay = INITIAL_BACKOFF;
|
||||
subscription
|
||||
}
|
||||
Err(_) => {
|
||||
wait_or_cancel(&cancellation, retry_delay).await;
|
||||
retry_delay = (retry_delay * 2).min(MAXIMUM_BACKOFF);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let next = subscription.next().fuse();
|
||||
pin_mut!(cancelled, next);
|
||||
let notified = select_biased! {
|
||||
_ = cancelled => return Ok(()),
|
||||
next = next => next,
|
||||
};
|
||||
match notified {
|
||||
Some(Ok(_)) => {
|
||||
let _ = self.publisher.refresh().await;
|
||||
}
|
||||
Some(Err(_)) | None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_or_cancel(cancellation: &CancellationToken, duration: Duration) {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let delay = Delay::new(duration).fuse();
|
||||
pin_mut!(cancelled, delay);
|
||||
select_biased! {
|
||||
_ = cancelled => {},
|
||||
_ = delay => {},
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前发布版本与持久版本不一致时必须重载;缺失和回退同样 fail closed
|
||||
#[must_use]
|
||||
pub fn runtime_revision_needs_refresh(
|
||||
published_revision: Option<u64>,
|
||||
persisted_revision: u64,
|
||||
) -> bool {
|
||||
published_revision != Some(persisted_revision)
|
||||
}
|
||||
|
||||
fn lock_unpoisoned<T>(lock: &SyncMutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
lock.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
@@ -1,17 +1,4 @@
|
||||
//! 冻结宿主基线与请求显式改写;派生快照只属于当前调用,不发布全局配置
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::account::FastMode;
|
||||
use crate::policy::{ClientApiKeyId, ClientPolicy, ClientSettings, RateLimits};
|
||||
use crate::routing::RuntimeSnapshot;
|
||||
//! 运行设置的纯值、校验与编译结果
|
||||
|
||||
pub(crate) mod compiled;
|
||||
mod values;
|
||||
@@ -19,365 +6,6 @@ pub use values::{
|
||||
SettingsValues, client_min_versions, response_body_limit, validate_request_limits,
|
||||
};
|
||||
|
||||
/// 一次模型调用的有效设置;改写只影响当前请求,不发布配置或修改持久化 revision
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionSettings {
|
||||
pub runtime: SettingsValues,
|
||||
pub fast_mode: FastMode,
|
||||
pub client_limits: RateLimits,
|
||||
/// null 表示不限制总时长;显式时限从请求开始计时,改写不重置计时原点
|
||||
pub timeout_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SettingOverride<T = Value> {
|
||||
pub instance_id: String,
|
||||
pub order: u64,
|
||||
pub value: T,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
struct ExecutionOverrides {
|
||||
#[serde(skip)]
|
||||
defaults: Arc<ClientSettings>,
|
||||
client_key_id: String,
|
||||
input: Arc<ExecutionSettings>,
|
||||
fast_mode: Option<SettingOverride<FastMode>>,
|
||||
client_limits: Option<SettingOverride<RateLimits>>,
|
||||
timeout_ms: Option<SettingOverride<Option<u64>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
struct HttpTimeout {
|
||||
input_ms: Option<u64>,
|
||||
change: Option<SettingOverride<Option<u64>>>,
|
||||
}
|
||||
|
||||
/// 共享值均不可变;同级子调用的修改不会回写父调用
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RequestSettings {
|
||||
baseline: Arc<RuntimeSnapshot>,
|
||||
effective: Arc<RuntimeSnapshot>,
|
||||
overrides: Arc<BTreeMap<String, SettingOverride>>,
|
||||
execution: Option<Arc<ExecutionOverrides>>,
|
||||
http_timeout: Option<HttpTimeout>,
|
||||
order: u64,
|
||||
}
|
||||
|
||||
impl PartialEq for RequestSettings {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
Arc::ptr_eq(&self.baseline, &other.baseline)
|
||||
&& self.overrides == other.overrides
|
||||
&& self.execution == other.execution
|
||||
&& self.http_timeout == other.http_timeout
|
||||
}
|
||||
}
|
||||
impl Eq for RequestSettings {}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("request settings are invalid")]
|
||||
pub struct InvalidSettings;
|
||||
|
||||
impl RequestSettings {
|
||||
#[must_use]
|
||||
pub fn new(snapshot: Arc<RuntimeSnapshot>) -> Self {
|
||||
Self {
|
||||
baseline: snapshot.clone(),
|
||||
effective: snapshot,
|
||||
overrides: Arc::default(),
|
||||
execution: None,
|
||||
http_timeout: None,
|
||||
order: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> Arc<RuntimeSnapshot> {
|
||||
self.effective.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn values(&self) -> &SettingsValues {
|
||||
self.effective.settings()
|
||||
}
|
||||
|
||||
/// 来源与有效值分开查询,不允许插件伪造其他实例的写入来源
|
||||
pub fn inspect(&self) -> Value {
|
||||
serde_json::json!({
|
||||
"config_revision": self.baseline.revision().get(),
|
||||
"host": self.baseline.settings(),
|
||||
"overrides": self.overrides,
|
||||
"execution": self.execution,
|
||||
"http_timeout": self.http_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn replace(
|
||||
&self,
|
||||
values: SettingsValues,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
self.replace_scoped(self.values(), values, instance_id)
|
||||
}
|
||||
|
||||
/// Key 画像等隐式作用域结果不记录成插件改写,避免随后切换 Key 时泄漏
|
||||
fn replace_scoped(
|
||||
&self,
|
||||
previous: &SettingsValues,
|
||||
values: SettingsValues,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
if previous == &values {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
let current = serde_json::to_value(previous).map_err(|_| InvalidSettings)?;
|
||||
let next = serde_json::to_value(&values).map_err(|_| InvalidSettings)?;
|
||||
let order = self.order.checked_add(1).ok_or(InvalidSettings)?;
|
||||
let mut overrides = self.overrides.as_ref().clone();
|
||||
let effective_values = self.values();
|
||||
// 每个已声明设置项整体替换,不猜测画像对象、数组或 null 的深合并含义
|
||||
for (name, value) in next.as_object().ok_or(InvalidSettings)? {
|
||||
if current.get(name) != Some(value) {
|
||||
overrides.insert(
|
||||
name.clone(),
|
||||
SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: value.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
// 只把本层实际修改的字段写回宿主层,作用域默认值不参与继承
|
||||
let values = if previous == effective_values {
|
||||
values
|
||||
} else {
|
||||
apply_overrides(self.baseline.settings(), &overrides)?
|
||||
};
|
||||
let effective = self
|
||||
.effective
|
||||
.resolve_settings(&values)
|
||||
.map_err(|_| InvalidSettings)?;
|
||||
Ok(Self {
|
||||
baseline: self.baseline.clone(),
|
||||
effective,
|
||||
overrides: Arc::new(overrides),
|
||||
execution: self.execution.clone(),
|
||||
http_timeout: self.http_timeout.clone(),
|
||||
order,
|
||||
})
|
||||
}
|
||||
|
||||
/// 持续会话的新请求读取当前宿主快照,只继承明确改写的字段
|
||||
pub fn rebase(&self, snapshot: Arc<RuntimeSnapshot>) -> Result<Self, InvalidSettings> {
|
||||
if Arc::ptr_eq(&self.baseline, &snapshot) {
|
||||
return Ok(self.clone());
|
||||
}
|
||||
let effective = if self.overrides.is_empty() {
|
||||
snapshot.clone()
|
||||
} else {
|
||||
snapshot
|
||||
.resolve_settings(&apply_overrides(snapshot.settings(), &self.overrides)?)
|
||||
.map_err(|_| InvalidSettings)?
|
||||
};
|
||||
Ok(Self {
|
||||
baseline: snapshot,
|
||||
effective,
|
||||
overrides: self.overrides.clone(),
|
||||
execution: self.execution.clone(),
|
||||
http_timeout: self.http_timeout.clone(),
|
||||
order: self.order,
|
||||
})
|
||||
}
|
||||
|
||||
/// 认证、模型入口和插件视图共用解析规则,派生策略不改变 Key 默认值
|
||||
#[must_use]
|
||||
pub fn apply_policy(&self, policy: ClientPolicy) -> ClientPolicy {
|
||||
let values = self.resolve_execution(policy.key_id().as_str(), policy.defaults(), None);
|
||||
policy.with_settings(
|
||||
values.runtime.request_profiles(),
|
||||
values.fast_mode,
|
||||
values.client_limits,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_execution(mut self, policy: &ClientPolicy, timeout_ms: Option<u64>) -> Self {
|
||||
let input = self.resolve_execution(policy.key_id().as_str(), policy.defaults(), timeout_ms);
|
||||
let previous = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == policy.key_id().as_str());
|
||||
self.execution = Some(Arc::new(ExecutionOverrides {
|
||||
defaults: policy.defaults().clone(),
|
||||
client_key_id: policy.key_id().as_str().to_owned(),
|
||||
input: Arc::new(input),
|
||||
fast_mode: previous.and_then(|scope| scope.fast_mode.clone()),
|
||||
client_limits: previous.and_then(|scope| scope.client_limits.clone()),
|
||||
timeout_ms: previous.and_then(|scope| scope.timeout_ms.clone()),
|
||||
}));
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn execution_values(&self) -> Option<ExecutionSettings> {
|
||||
let scope = self.execution.as_ref()?;
|
||||
Some(self.resolve_execution(
|
||||
&scope.client_key_id,
|
||||
&scope.defaults,
|
||||
scope.input.timeout_ms,
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_execution(
|
||||
&self,
|
||||
key: &str,
|
||||
defaults: &ClientSettings,
|
||||
timeout_ms: Option<u64>,
|
||||
) -> ExecutionSettings {
|
||||
let overrides = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key);
|
||||
let mut runtime = self.values().clone();
|
||||
// Key 画像覆盖宿主默认,插件显式覆盖整个设置项;此优先级只在这里解释
|
||||
if !self.overrides.contains_key("request_profiles") && !defaults.request_profiles.is_empty()
|
||||
{
|
||||
let mut profiles = runtime.request_profiles().clone();
|
||||
profiles.extend(defaults.request_profiles.clone());
|
||||
runtime = runtime.with_request_profiles(profiles);
|
||||
}
|
||||
ExecutionSettings {
|
||||
runtime,
|
||||
fast_mode: overrides
|
||||
.and_then(|scope| scope.fast_mode.as_ref())
|
||||
.map_or(defaults.fast_mode, |change| change.value),
|
||||
client_limits: overrides
|
||||
.and_then(|scope| scope.client_limits.as_ref())
|
||||
.map_or(defaults.limits, |change| change.value),
|
||||
timeout_ms: overrides
|
||||
.and_then(|scope| scope.timeout_ms.as_ref())
|
||||
.map_or(timeout_ms, |change| change.value),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn execution_deadline(
|
||||
&self,
|
||||
key: &ClientApiKeyId,
|
||||
started_at: SystemTime,
|
||||
) -> Result<crate::lifecycle::Deadline, InvalidSettings> {
|
||||
let scope = self
|
||||
.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key.as_str())
|
||||
.ok_or(InvalidSettings)?;
|
||||
let timeout_ms = scope
|
||||
.timeout_ms
|
||||
.as_ref()
|
||||
.map_or(scope.input.timeout_ms, |change| change.value);
|
||||
crate::lifecycle::Deadline::from_timeout(started_at, timeout_ms.map(Duration::from_millis))
|
||||
.ok_or(InvalidSettings)
|
||||
}
|
||||
|
||||
pub fn replace_execution(
|
||||
&self,
|
||||
values: &ExecutionSettings,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
let previous = self.execution_values().ok_or(InvalidSettings)?;
|
||||
let mut updated =
|
||||
self.replace_scoped(&previous.runtime, values.runtime.clone(), instance_id)?;
|
||||
if previous.fast_mode == values.fast_mode
|
||||
&& previous.client_limits == values.client_limits
|
||||
&& previous.timeout_ms == values.timeout_ms
|
||||
{
|
||||
return Ok(updated);
|
||||
}
|
||||
let mut scope = self.execution.as_deref().cloned().ok_or(InvalidSettings)?;
|
||||
let order = if updated.order == self.order {
|
||||
self.order.checked_add(1).ok_or(InvalidSettings)?
|
||||
} else {
|
||||
updated.order
|
||||
};
|
||||
if previous.fast_mode != values.fast_mode {
|
||||
scope.fast_mode = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.fast_mode,
|
||||
});
|
||||
}
|
||||
if previous.client_limits != values.client_limits {
|
||||
scope.client_limits = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.client_limits,
|
||||
});
|
||||
}
|
||||
if previous.timeout_ms != values.timeout_ms {
|
||||
scope.timeout_ms = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order,
|
||||
value: values.timeout_ms,
|
||||
});
|
||||
}
|
||||
updated.execution = Some(Arc::new(scope));
|
||||
updated.order = order;
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn execution_timeout(&self, key: &ClientApiKeyId) -> Option<Duration> {
|
||||
self.execution
|
||||
.as_ref()
|
||||
.filter(|scope| scope.client_key_id == key.as_str())
|
||||
.and_then(|scope| scope.timeout_ms.as_ref())
|
||||
.and_then(|change| change.value.map(Duration::from_millis))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_http_timeout(mut self, timeout_ms: Option<u64>) -> Self {
|
||||
self.http_timeout.get_or_insert(HttpTimeout {
|
||||
input_ms: timeout_ms,
|
||||
change: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
pub fn replace_http_timeout(
|
||||
mut self,
|
||||
timeout_ms: Option<u64>,
|
||||
instance_id: &str,
|
||||
) -> Result<Self, InvalidSettings> {
|
||||
let timeout = self.http_timeout.get_or_insert(HttpTimeout {
|
||||
input_ms: None,
|
||||
change: None,
|
||||
});
|
||||
if timeout
|
||||
.change
|
||||
.as_ref()
|
||||
.map_or(timeout.input_ms, |change| change.value)
|
||||
!= timeout_ms
|
||||
{
|
||||
self.order = self.order.checked_add(1).ok_or(InvalidSettings)?;
|
||||
timeout.change = Some(SettingOverride {
|
||||
instance_id: instance_id.to_owned(),
|
||||
order: self.order,
|
||||
value: timeout_ms,
|
||||
});
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_overrides(
|
||||
values: &SettingsValues,
|
||||
overrides: &BTreeMap<String, SettingOverride>,
|
||||
) -> Result<SettingsValues, InvalidSettings> {
|
||||
let mut values = serde_json::to_value(values).map_err(|_| InvalidSettings)?;
|
||||
let object = values.as_object_mut().ok_or(InvalidSettings)?;
|
||||
for (name, change) in overrides {
|
||||
object.insert(name.clone(), change.value.clone());
|
||||
}
|
||||
serde_json::from_value(values).map_err(|_| InvalidSettings)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 上游调用跨 Engine、Event、Error 与 Provider 共享的中立边界事实
|
||||
|
||||
use std::fmt;
|
||||
use std::{fmt, num::NonZeroU32};
|
||||
|
||||
use crate::validation::{IdentifierError, validate_text};
|
||||
|
||||
@@ -73,3 +73,15 @@ impl fmt::Debug for OpaqueUpstreamValue {
|
||||
formatter.write_str("OpaqueUpstreamValue(<redacted-from-Debug>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 可为当前 attempt 选择的请求局部传输档位
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
pub enum AttemptTransport {
|
||||
/// 使用 Provider 的默认传输策略
|
||||
#[default]
|
||||
Default,
|
||||
/// 固定账号重试 Provider 的首选传输;序号由 Provider 的独立预算驱动
|
||||
Retry(NonZeroU32),
|
||||
/// 使用 Provider 定义的备用传输
|
||||
Fallback,
|
||||
}
|
||||
|
||||
+10
-10
@@ -1,4 +1,5 @@
|
||||
//! 执行服务的准入、预算、请求记录与终结行为测试
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, VecDeque},
|
||||
@@ -2964,10 +2965,7 @@ use gateway_core::routing::{
|
||||
PublicModelId, RoutingGroupSnapshot, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
use gateway_core::runtime::{
|
||||
RuntimeSnapshotHandle,
|
||||
extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
use gateway_core::runtime::RuntimeSnapshotHandle;
|
||||
use gateway_core::upstream::{UpstreamSendState, UpstreamTransport};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -3218,7 +3216,7 @@ impl Provider for NativeCatalogProvider {
|
||||
fn client_catalog_forwards_scope_maps_whole_objects_and_omits_unroutable_models() {
|
||||
#[derive(Debug)]
|
||||
struct Aliases(Vec<gateway_core::routing::ContributedModelAlias>);
|
||||
impl gateway_core::runtime::extensions::ExtensionSetLease for Aliases {
|
||||
impl gateway_core::routing::extensions::ExtensionSetLease for Aliases {
|
||||
fn is_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
@@ -3239,7 +3237,7 @@ fn client_catalog_forwards_scope_maps_whole_objects_and_omits_unroutable_models(
|
||||
.with_settings(&settings)
|
||||
.unwrap()
|
||||
.with_extensions(Some(ExtensionSetReference::new(
|
||||
gateway_core::runtime::extensions::ExtensionSetId::new("catalog-aliases".into())
|
||||
gateway_core::routing::extensions::ExtensionSetId::new("catalog-aliases".into())
|
||||
.unwrap(),
|
||||
Arc::new(Aliases(vec![
|
||||
gateway_core::routing::ContributedModelAlias {
|
||||
@@ -4500,7 +4498,7 @@ fn invalid_request_settings_leave_the_prepared_execution_unchanged() {
|
||||
|
||||
#[test]
|
||||
fn execution_settings_must_match_the_prepared_key() {
|
||||
use gateway_core::settings::RequestSettings;
|
||||
use gateway_core::routing::request_settings::RequestSettings;
|
||||
|
||||
let service = service(Arc::new(Admissions::default()), Arc::new(Budget::default()));
|
||||
let client = service.authenticate("sk_start_test").unwrap();
|
||||
@@ -4595,7 +4593,7 @@ fn repeated_connection_failures_never_block_later_requests_for_the_provider() {
|
||||
#[test]
|
||||
fn entry_settings_freeze_authentication_and_rebase_only_explicit_overrides() {
|
||||
block_on(async {
|
||||
use gateway_core::settings::RequestSettings;
|
||||
use gateway_core::routing::request_settings::RequestSettings;
|
||||
let snapshots = RuntimeSnapshotHandle::new(start_snapshot());
|
||||
let service = DefaultExecutionService::new(
|
||||
snapshots.clone(),
|
||||
@@ -4707,7 +4705,7 @@ fn entry_settings_freeze_authentication_and_rebase_only_explicit_overrides() {
|
||||
|
||||
#[test]
|
||||
fn unchanged_and_precompiled_request_settings_reuse_the_frozen_snapshot() {
|
||||
use gateway_core::settings::RequestSettings;
|
||||
use gateway_core::routing::request_settings::RequestSettings;
|
||||
|
||||
let snapshot = Arc::new(start_snapshot());
|
||||
let baseline = RequestSettings::new(snapshot.clone());
|
||||
@@ -4785,7 +4783,9 @@ fn unchanged_and_precompiled_request_settings_reuse_the_frozen_snapshot() {
|
||||
#[test]
|
||||
fn child_settings_recompute_key_scope_without_inheriting_parent_defaults() {
|
||||
block_on(async {
|
||||
use gateway_core::{account::OpaqueProviderData, settings::RequestSettings};
|
||||
use gateway_core::{
|
||||
account::OpaqueProviderData, routing::request_settings::RequestSettings,
|
||||
};
|
||||
let provider = ProviderKind::new("openai").unwrap();
|
||||
let profiles = |name: &str| {
|
||||
BTreeMap::from([(
|
||||
@@ -1,8 +1,13 @@
|
||||
//! 执行测试共用的扩展代次引用与就绪租约替身
|
||||
//! 执行扩展索引的代次隔离、计划保活与租约替身
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_core::runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::engine::extensions::{ExecutionExtensionIndex, ExecutionExtensionPlans};
|
||||
use gateway_core::engine::upstream_adapter::{UpstreamAdapter, UpstreamAdapterPlan};
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::{
|
||||
engine::AttemptContext, error::ProviderError, identity::ProviderKind, routing::UpstreamModelId,
|
||||
};
|
||||
|
||||
struct ReadyLease;
|
||||
|
||||
@@ -16,3 +21,71 @@ pub(super) fn reference(id: &str) -> ExtensionSetReference {
|
||||
let id = ExtensionSetId::new(id.into()).unwrap();
|
||||
ExtensionSetReference::new(id, Arc::new(ReadyLease))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct EmptyAdapterPlan;
|
||||
|
||||
impl UpstreamAdapterPlan for EmptyAdapterPlan {
|
||||
fn select(
|
||||
&self,
|
||||
_: &AttemptContext,
|
||||
_: &ProviderKind,
|
||||
_: &UpstreamModelId,
|
||||
) -> Result<Option<Arc<dyn UpstreamAdapter>>, ProviderError> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
struct PublishedPlans(Arc<ExecutionExtensionPlans>);
|
||||
|
||||
impl ExtensionSetLease for PublishedPlans {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.0.upstream_adapters.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adapter_only_generation_stays_alive_until_the_last_frozen_plan_is_released() {
|
||||
let index = ExecutionExtensionIndex::default();
|
||||
let id = ExtensionSetId::new("adapter-only".into()).unwrap();
|
||||
let plans = Arc::new(ExecutionExtensionPlans {
|
||||
middleware: None,
|
||||
upstream_adapters: Some(Arc::new(EmptyAdapterPlan)),
|
||||
});
|
||||
let weak_plans = Arc::downgrade(&plans);
|
||||
let published = Arc::new(PublishedPlans(index.register(id.clone(), plans).unwrap()));
|
||||
let weak_publication = Arc::downgrade(&published);
|
||||
let reference = ExtensionSetReference::new(id.clone(), published);
|
||||
assert!(index.middleware(&reference).is_none());
|
||||
let frozen = index.upstream_adapters(&reference).unwrap();
|
||||
drop(reference);
|
||||
|
||||
// 当前发布可以退场;在途计划仍保活整个代次,不能覆盖相同身份
|
||||
assert!(weak_publication.upgrade().is_some());
|
||||
assert!(weak_plans.upgrade().is_some());
|
||||
assert!(index.register(id.clone(), Arc::default()).is_err());
|
||||
let in_flight = frozen.clone();
|
||||
drop(frozen);
|
||||
assert!(weak_publication.upgrade().is_some());
|
||||
drop(in_flight);
|
||||
|
||||
assert!(weak_publication.upgrade().is_none());
|
||||
assert!(weak_plans.upgrade().is_none());
|
||||
let released = ExtensionSetReference::new(id.clone(), Arc::new(ReadyLease));
|
||||
assert!(index.upstream_adapters(&released).is_none());
|
||||
let _replacement = index.register(id, Arc::default()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_index_does_not_own_registered_generations() {
|
||||
let index = ExecutionExtensionIndex::default();
|
||||
let reference = reference("unpublished");
|
||||
let plans = Arc::new(ExecutionExtensionPlans::default());
|
||||
let weak = Arc::downgrade(&plans);
|
||||
let owner = index.register(reference.id().clone(), plans).unwrap();
|
||||
assert!(weak.upgrade().is_some());
|
||||
drop(owner);
|
||||
assert!(weak.upgrade().is_none());
|
||||
assert!(index.middleware(&reference).is_none());
|
||||
assert!(index.upstream_adapters(&reference).is_none());
|
||||
}
|
||||
|
||||
+18
-10
@@ -12,11 +12,13 @@ use gateway_core::account::ProviderAccountId;
|
||||
use gateway_core::account::scope::AccountGroupId;
|
||||
use gateway_core::engine::ModelRequestId;
|
||||
use gateway_core::engine::execution::ClientTransport;
|
||||
use gateway_core::engine::extensions::ExtensionCallScope;
|
||||
use gateway_core::engine::extensions::{
|
||||
ExecutionExtensionIndex, ExecutionExtensionPlans, ExtensionCallScope,
|
||||
};
|
||||
use gateway_core::engine::middleware::{
|
||||
FrozenMiddlewarePlan, MiddlewareAuthority, MiddlewareBody, MiddlewareContext, MiddlewareError,
|
||||
MiddlewareExtensionIndex, MiddlewareFrame, MiddlewareHeader, MiddlewareMount, MiddlewareNext,
|
||||
MiddlewarePlan, MiddlewareRequest, MiddlewareResponse, MiddlewareTarget,
|
||||
MiddlewareFrame, MiddlewareHeader, MiddlewareMount, MiddlewareNext, MiddlewarePlan,
|
||||
MiddlewareRequest, MiddlewareResponse, MiddlewareTarget,
|
||||
};
|
||||
use gateway_core::engine::provider::{
|
||||
ProviderCallMetadata, ProviderMiddlewareTerminal, ProviderStream, execute_attempt_middleware,
|
||||
@@ -27,7 +29,7 @@ use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
use gateway_core::upstream::UpstreamTransport;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -303,14 +305,17 @@ fn attempt_middleware_consumes_owned_next_once_and_preserves_host_envelopes() {
|
||||
futures::executor::block_on(async {
|
||||
let plan_calls = Arc::new(AtomicUsize::new(0));
|
||||
let terminal_calls = Arc::new(AtomicUsize::new(0));
|
||||
let index = MiddlewareExtensionIndex::default();
|
||||
let index = ExecutionExtensionIndex::default();
|
||||
let id = ExtensionSetId::new("middleware-generation".to_owned()).unwrap();
|
||||
let reference = ExtensionSetReference::new(id.clone(), Arc::new(ReadyGeneration));
|
||||
let owner = index
|
||||
.register(
|
||||
id.clone(),
|
||||
Arc::new(RewritingPlan {
|
||||
calls: Arc::clone(&plan_calls),
|
||||
Arc::new(ExecutionExtensionPlans {
|
||||
middleware: Some(Arc::new(RewritingPlan {
|
||||
calls: Arc::clone(&plan_calls),
|
||||
})),
|
||||
upstream_adapters: None,
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -318,14 +323,17 @@ fn attempt_middleware_consumes_owned_next_once_and_preserves_host_envelopes() {
|
||||
index
|
||||
.register(
|
||||
id,
|
||||
Arc::new(RewritingPlan {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
Arc::new(ExecutionExtensionPlans {
|
||||
middleware: Some(Arc::new(RewritingPlan {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
})),
|
||||
upstream_adapters: None
|
||||
}),
|
||||
)
|
||||
.is_err(),
|
||||
"同一发布代次不能静默替换中间件计划",
|
||||
);
|
||||
let plan = index.resolve(&reference).unwrap();
|
||||
let plan = index.middleware(&reference).unwrap();
|
||||
|
||||
let terminal: ProviderMiddlewareTerminal = {
|
||||
let terminal_calls = Arc::clone(&terminal_calls);
|
||||
@@ -24,7 +24,7 @@ use gateway_core::{
|
||||
},
|
||||
policy::ClientApiKeyId,
|
||||
routing::ProviderKind,
|
||||
runtime::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -16,7 +16,6 @@ mod policy;
|
||||
mod provider_ports;
|
||||
mod routing;
|
||||
mod runtime;
|
||||
mod settings;
|
||||
mod task;
|
||||
mod upstream;
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ use gateway_core::routing::{
|
||||
UpstreamModelId,
|
||||
};
|
||||
|
||||
mod request_settings;
|
||||
mod snapshot;
|
||||
|
||||
fn settings() -> gateway_core::settings::SettingsValues {
|
||||
|
||||
+2
-1
@@ -7,8 +7,9 @@ use gateway_core::{
|
||||
account::OpaqueProviderData,
|
||||
identity::ProviderKind,
|
||||
policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits},
|
||||
routing::request_settings::RequestSettings,
|
||||
routing::{ClientRoutingScope, ConfigRevision, FrozenAccountScope, RuntimeSnapshot},
|
||||
settings::{RequestSettings, SettingsValues},
|
||||
settings::SettingsValues,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -10,6 +10,10 @@ use futures::future::BoxFuture;
|
||||
use gateway_core::account::FastMode;
|
||||
use gateway_core::operation::OperationKind;
|
||||
use gateway_core::policy::{ClientApiKeyId, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::extensions::{
|
||||
ExtensionPreparationError, ExtensionPreparationPort, ExtensionSetId, ExtensionSetLease,
|
||||
ExtensionSetReference,
|
||||
};
|
||||
use gateway_core::routing::snapshot::{
|
||||
RuntimeSnapshotCompileError, RuntimeSnapshotCompiler, SnapshotAccountGroupFacts,
|
||||
SnapshotAccountGroupMemberFacts, SnapshotClientPolicyFacts, SnapshotFacts,
|
||||
@@ -20,10 +24,6 @@ use gateway_core::routing::{
|
||||
ProviderCatalogGeneration, ProviderCatalogPort, ProviderCatalogUnavailable, ProviderKind,
|
||||
ProviderModelCapabilities, PublicModelId, UpstreamModelId,
|
||||
};
|
||||
use gateway_core::runtime::extensions::{
|
||||
ExtensionPreparationError, ExtensionPreparationPort, ExtensionSetId, ExtensionSetLease,
|
||||
ExtensionSetReference,
|
||||
};
|
||||
use gateway_core::settings::SettingsValues;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -1084,7 +1084,9 @@ fn key_profiles_replace_whole_global_choice_and_previous_snapshot_stays_frozen()
|
||||
.unwrap()
|
||||
};
|
||||
let values = |snapshot: &gateway_core::routing::RuntimeSnapshot| {
|
||||
let settings = gateway_core::settings::RequestSettings::new(Arc::new(snapshot.clone()));
|
||||
let settings = gateway_core::routing::request_settings::RequestSettings::new(Arc::new(
|
||||
snapshot.clone(),
|
||||
));
|
||||
let mut values: Vec<_> = snapshot
|
||||
.client_policies()
|
||||
.map(|policy| {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 验证运行时快照发布、版本对账与并发更新行为
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
mod extensions;
|
||||
mod publications;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
|
||||
+3
-3
@@ -5,9 +5,9 @@ use std::sync::{
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use gateway_core::runtime::{
|
||||
RuntimeSnapshotHandle,
|
||||
extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
use gateway_core::{
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference},
|
||||
runtime::RuntimeSnapshotHandle,
|
||||
};
|
||||
|
||||
struct Lease {
|
||||
@@ -59,7 +59,9 @@ pub async fn initialize(config: HostConfig) -> Result<HostBundle, HostError> {
|
||||
let log_guard = initialize_logging(&config.logging, config.timezone)?;
|
||||
let cancellation = CancellationToken::new();
|
||||
let connections = Arc::new(ConnectionTracker::new(cancellation.clone()));
|
||||
let workers = WorkerSupervisor::new(cancellation.clone());
|
||||
// 关闭请求先排空 HTTP/WS,写泵仍需接收在途请求的最后写入
|
||||
// Worker 使用独立信号,只由 drain 结束后的 shutdown 或析构触发
|
||||
let workers = WorkerSupervisor::new(CancellationToken::new());
|
||||
let system = Arc::new(ProcessSystemOperations::new(
|
||||
cancellation.clone(),
|
||||
config.system_update.clone(),
|
||||
@@ -190,6 +192,14 @@ impl HostBundle {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for HostBundle {
|
||||
fn drop(&mut self) {
|
||||
// CLI、启动失败和 serve future 被丢弃也要通知公共能力停止
|
||||
// WorkerSupervisor 自己负责取消并终止剩余任务
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum HostError {
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
//! 系统更新的部署配置、编译元数据与本地路径解析
|
||||
|
||||
use std::{
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{OperationError, conflict, internal};
|
||||
use crate::config::ConfigError;
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const DEFAULT_GITHUB_API_BASE: &str = "https://api.github.com/repos";
|
||||
const DEFAULT_UPDATE_REPOSITORY: &str = "zyycn/codex-proxy-rs";
|
||||
|
||||
/// 系统更新与重启配置;所有字段只由 Host 解释
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct SystemUpdateConfig {
|
||||
pub version: String,
|
||||
pub git_sha: String,
|
||||
pub build_time: String,
|
||||
pub deployment_mode: String,
|
||||
pub build_type: String,
|
||||
pub update_repository: Option<String>,
|
||||
pub github_api_base: String,
|
||||
pub executable_path: Option<PathBuf>,
|
||||
/// 未显式指定时,由组合根传入 API 实际使用的静态资源目录
|
||||
pub web_dist_dir: Option<PathBuf>,
|
||||
pub update_state_file: PathBuf,
|
||||
pub update_lock_file: PathBuf,
|
||||
pub update_temp_dir: PathBuf,
|
||||
pub self_restart_enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for SystemUpdateConfig {
|
||||
fn default() -> Self {
|
||||
let deployment_mode =
|
||||
environment_value("CPR_DEPLOYMENT_MODE").unwrap_or_else(|| "source".to_owned());
|
||||
let executable_path = environment_value("CPR_UPDATE_EXE_PATH")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| {
|
||||
(deployment_mode == "docker")
|
||||
.then(|| PathBuf::from("/app/bin").join(APP_BINARY_NAME))
|
||||
});
|
||||
let update_state_file = environment_value("CPR_UPDATE_STATE_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("update-state.json"));
|
||||
let update_lock_file = environment_value("CPR_UPDATE_LOCK_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| update_state_file.with_file_name("update.lock"));
|
||||
let update_temp_dir = environment_value("CPR_UPDATE_TEMP_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| default_temp_dir(&update_state_file));
|
||||
Self {
|
||||
version: option_env!("CPR_VERSION")
|
||||
.unwrap_or(env!("CARGO_PKG_VERSION"))
|
||||
.to_owned(),
|
||||
git_sha: option_env!("CPR_GIT_SHA").unwrap_or("unknown").to_owned(),
|
||||
build_time: option_env!("CPR_BUILD_TIME")
|
||||
.unwrap_or("unknown")
|
||||
.to_owned(),
|
||||
deployment_mode,
|
||||
build_type: option_env!("CPR_BUILD_TYPE").unwrap_or("source").to_owned(),
|
||||
update_repository: Some(
|
||||
environment_value("CPR_UPDATE_REPOSITORY")
|
||||
.unwrap_or_else(|| DEFAULT_UPDATE_REPOSITORY.to_owned()),
|
||||
),
|
||||
github_api_base: environment_value("CPR_GITHUB_API_BASE")
|
||||
.unwrap_or_else(|| DEFAULT_GITHUB_API_BASE.to_owned()),
|
||||
executable_path,
|
||||
web_dist_dir: None,
|
||||
update_state_file,
|
||||
update_lock_file,
|
||||
update_temp_dir,
|
||||
self_restart_enabled: environment_value("CPR_ENABLE_SELF_RESTART").as_deref()
|
||||
== Some("true"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SystemUpdateConfig {
|
||||
pub(crate) fn resolve_and_validate(
|
||||
&mut self,
|
||||
source_dir: &Path,
|
||||
runtime_data_dir: &Path,
|
||||
asset_directory: &Path,
|
||||
) -> Result<(), ConfigError> {
|
||||
let web_dist_dir = self
|
||||
.web_dist_dir
|
||||
.get_or_insert_with(|| asset_directory.to_path_buf());
|
||||
if web_dist_dir.as_os_str().is_empty() {
|
||||
return Err(ConfigError::InvalidField("host.system_update.web_dist_dir"));
|
||||
}
|
||||
for path in [self.executable_path.as_mut(), self.web_dist_dir.as_mut()]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if path.is_relative() {
|
||||
*path = source_dir.join(&*path);
|
||||
}
|
||||
}
|
||||
for path in [
|
||||
&mut self.update_state_file,
|
||||
&mut self.update_lock_file,
|
||||
&mut self.update_temp_dir,
|
||||
] {
|
||||
if path.is_relative() {
|
||||
*path = runtime_data_dir.join(&*path);
|
||||
}
|
||||
}
|
||||
if self.version.trim().is_empty() {
|
||||
return Err(ConfigError::InvalidField("host.system_update.version"));
|
||||
}
|
||||
if !matches!(
|
||||
self.deployment_mode.as_str(),
|
||||
"source" | "binary" | "docker"
|
||||
) {
|
||||
return Err(ConfigError::InvalidField(
|
||||
"host.system_update.deployment_mode",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn executable_path(&self) -> Result<PathBuf, OperationError> {
|
||||
if let Some(path) = &self.executable_path {
|
||||
return Ok(path.clone());
|
||||
}
|
||||
env::current_exe()
|
||||
.and_then(fs::canonicalize)
|
||||
.map_err(|error| internal(format!("failed to resolve executable: {error}")))
|
||||
}
|
||||
|
||||
pub(crate) fn official_plugins_dir(&self) -> Result<PathBuf, OperationError> {
|
||||
let executable = self.executable_path()?;
|
||||
let parent = executable
|
||||
.parent()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.ok_or_else(|| internal("failed to resolve official plugin release directory"))?;
|
||||
Ok(parent.join("plugins").join("official"))
|
||||
}
|
||||
|
||||
pub(crate) fn web_dist_dir(&self) -> Result<&Path, OperationError> {
|
||||
self.web_dist_dir
|
||||
.as_deref()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.ok_or_else(|| conflict("web assets directory is not configured"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn environment_value(key: &str) -> Option<String> {
|
||||
env::var(key)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn default_temp_dir(state_file: &Path) -> PathBuf {
|
||||
state_file
|
||||
.parent()
|
||||
.map(|parent| parent.join("update-tmp"))
|
||||
.unwrap_or_else(|| std::env::temp_dir().join("codex-proxy-rs-update"))
|
||||
}
|
||||
@@ -8,8 +8,9 @@ use std::time::Duration;
|
||||
use futures::StreamExt as _;
|
||||
use sha2::Digest as _;
|
||||
|
||||
use super::events::UpdateEvents;
|
||||
use super::release::{download_client, validate_download_url};
|
||||
use super::{OperationError, UpdateEvents, invalid, upstream};
|
||||
use super::{OperationError, invalid, upstream};
|
||||
|
||||
pub(crate) const MAX_DOWNLOAD_SIZE: u64 = 500 * 1024 * 1024;
|
||||
pub(crate) const MAX_CHECKSUM_SIZE: u64 = 1024 * 1024;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user