diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 43bad2e09..959aee5b8 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -3289,6 +3289,7 @@ dependencies = [ "native-tls", "openssl", "openssl-sys", + "regex", "reqwest", "roxmltree", "rustls", diff --git a/backend/crates/gateway-admin/src/model/settings.rs b/backend/crates/gateway-admin/src/model/settings.rs index 5a5e7d6df..bc587685e 100644 --- a/backend/crates/gateway-admin/src/model/settings.rs +++ b/backend/crates/gateway-admin/src/model/settings.rs @@ -45,6 +45,7 @@ pub use gateway_core::account::RotationStrategy; /// 运行设置的共同值；版本、Provider 画像更新和数据库秘密由外层类型拥有 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct RuntimeSettingsValues { + pub codex_privacy_policy: gateway_core::settings::privacy::CodexPrivacyPolicy, pub request_location_enabled: bool, pub request_location: gateway_core::account::RequestLocation, pub refresh_margin_seconds: u64, diff --git a/backend/crates/gateway-admin/src/ports/provider.rs b/backend/crates/gateway-admin/src/ports/provider.rs index 0079f2610..a34c3e570 100644 --- a/backend/crates/gateway-admin/src/ports/provider.rs +++ b/backend/crates/gateway-admin/src/ports/provider.rs @@ -130,6 +130,32 @@ struct ProviderAdminDiagnostic { /// 运行时资源通知只在事务成功后发生 #[async_trait] pub trait ProviderAdmin: Send + Sync { + fn compile_privacy_policy( + &self, + _policy: &gateway_core::settings::privacy::CodexPrivacyPolicy, + ) -> Result< + Arc, + gateway_core::settings::privacy::PrivacyError, + > { + Err(gateway_core::settings::privacy::PrivacyError { + rule_index: 0, + reason: "Provider 不支持隐私策略", + }) + } + + fn preview_privacy_policy( + &self, + _request: gateway_core::settings::privacy::PrivacyPreviewRequest, + ) -> Result< + gateway_core::settings::privacy::PrivacyPreviewResult, + gateway_core::settings::privacy::PrivacyError, + > { + Err(gateway_core::settings::privacy::PrivacyError { + rule_index: 0, + reason: "Provider 不支持隐私策略", + }) + } + /// 只读内置价目；没有本地计价能力的 Provider 返回空目录 fn pricing_catalog(&self) -> crate::model::pricing::ProviderPricingCatalog { Default::default() diff --git a/backend/crates/gateway-admin/src/public_service/settings.rs b/backend/crates/gateway-admin/src/public_service/settings.rs index fa768f68e..5d667a816 100644 --- a/backend/crates/gateway-admin/src/public_service/settings.rs +++ b/backend/crates/gateway-admin/src/public_service/settings.rs @@ -25,6 +25,13 @@ pub(super) fn register( }}; } register!(Load, load, (), ()); + register!( + PreviewPrivacyPolicy, + preview_privacy_policy, + gateway_core::settings::privacy::PrivacyPreviewRequest, + request, + request + ); register!( Replace, replace, diff --git a/backend/crates/gateway-admin/src/use_case/settings.rs b/backend/crates/gateway-admin/src/use_case/settings.rs index edd181c4a..fd7ef8a58 100644 --- a/backend/crates/gateway-admin/src/use_case/settings.rs +++ b/backend/crates/gateway-admin/src/use_case/settings.rs @@ -22,6 +22,13 @@ use super::{map_store_error, publish_committed}; /// API 消费的 Runtime settings 管理服务 #[async_trait] pub trait SettingsService: Send + Sync { + async fn preview_privacy_policy( + &self, + _request: gateway_core::settings::privacy::PrivacyPreviewRequest, + ) -> Result { + Err(AdminError::invalid("隐私预览不可用")) + } + async fn preview_pricing_sync( &self, ) -> Result; @@ -117,6 +124,21 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> #[async_trait] impl SettingsService for DefaultSettingsService { + async fn preview_privacy_policy( + &self, + request: gateway_core::settings::privacy::PrivacyPreviewRequest, + ) -> Result { + self.profile_provider("openai")? + .preview_privacy_policy(request) + .map_err(|error| { + AdminError::invalid(format!( + "第 {} 条规则：{}", + error.rule_index + 1, + error.reason + )) + }) + } + async fn preview_pricing_sync( &self, ) -> Result { @@ -342,6 +364,17 @@ impl SettingsService for DefaultSettingsService { mut command: ReplaceRuntimeSettings, ) -> Result { validate_settings(&command)?; + if command.values.codex_privacy_policy != Default::default() { + self.profile_provider("openai")? + .compile_privacy_policy(&command.values.codex_privacy_policy) + .map_err(|error| { + AdminError::invalid(format!( + "第 {} 条规则：{}", + error.rule_index + 1, + error.reason + )) + })?; + } if command .request_profile_updates .values() diff --git a/backend/crates/gateway-admin/tests/use_case/accounts.rs b/backend/crates/gateway-admin/tests/use_case/accounts.rs index 4dd3e0589..97e38a45f 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts.rs @@ -1115,6 +1115,7 @@ impl SettingsStore for StaticSettingsStore { rotation_strategy: RotationStrategy::Smart, updated_at: Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 300, diff --git a/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs b/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs index 851352d35..8d56f77b7 100644 --- a/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs +++ b/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs @@ -32,6 +32,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> Runti rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart, updated_at: Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 300, diff --git a/backend/crates/gateway-admin/tests/use_case/observability.rs b/backend/crates/gateway-admin/tests/use_case/observability.rs index 29b6150dd..646d74c1e 100644 --- a/backend/crates/gateway-admin/tests/use_case/observability.rs +++ b/backend/crates/gateway-admin/tests/use_case/observability.rs @@ -988,6 +988,7 @@ impl SettingsStore for FixtureSettingsStore { rotation_strategy: RotationStrategy::Smart, updated_at: Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 300, diff --git a/backend/crates/gateway-admin/tests/use_case/service_contract.rs b/backend/crates/gateway-admin/tests/use_case/service_contract.rs index 35cb471d7..426605239 100644 --- a/backend/crates/gateway-admin/tests/use_case/service_contract.rs +++ b/backend/crates/gateway-admin/tests/use_case/service_contract.rs @@ -57,6 +57,7 @@ impl VisitMut for WireTypes { let last = path.path.segments.last().unwrap().clone(); *ty = match last.ident.to_string().as_str() { "DateTime" | "Tz" | "TokenPrice" | "AdminApiKey" => parse_quote!(String), + "Value" => parse_quote!(serde_json::Value), "OpaqueProviderData" => parse_quote!(serde_json::Map), _ => Type::Path(syn::TypePath { attrs: std::mem::take(&mut path.attrs), @@ -79,6 +80,10 @@ fn contract_type(mut item: Item, name: &str) -> Item { | "ReplaceRuntimeSettings" | "AdminApiKeyMutation" | "RegeneratedAdminApiKey" + | "CodexPrivacyPolicy" + | "PrivacyRule" + | "PrivacyPreviewRequest" + | "PrivacyPreviewResult" ); ( &mut item.attrs, @@ -107,6 +112,13 @@ fn contract_type(mut item: Item, name: &str) -> Item { field.vis = parse_quote!(pub); } } + if let Item::Enum(item) = &mut item { + for variant in &mut item.variants { + variant + .attrs + .retain(|attr| !attr.path().is_ident("default")); + } + } WireTypes.visit_item_mut(&mut item); item } @@ -117,6 +129,19 @@ fn sdk_settings_contract_matches_host_declarations() { let mut types = Vec::new(); let mut replacement_fields = Vec::new(); for (path, names) in [ + ( + "../gateway-core/src/settings/privacy.rs", + &[ + "CodexPrivacyPolicy", + "PrivacyFailureMode", + "PrivacyScope", + "PrivacyAction", + "PrivacyRule", + "PrivacyRuleOutcome", + "PrivacyPreviewRequest", + "PrivacyPreviewResult", + ][..], + ), ( "src/model/settings.rs", &[ diff --git a/backend/crates/gateway-admin/tests/use_case/settings.rs b/backend/crates/gateway-admin/tests/use_case/settings.rs index 517ca0448..db30fdd77 100644 --- a/backend/crates/gateway-admin/tests/use_case/settings.rs +++ b/backend/crates/gateway-admin/tests/use_case/settings.rs @@ -280,6 +280,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() { model_mappings: Default::default(), rotation_strategy: RotationStrategy::Smart, values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 0, diff --git a/backend/crates/gateway-api/src/admin/settings.rs b/backend/crates/gateway-api/src/admin/settings.rs index b6e706a18..78ff8c370 100644 --- a/backend/crates/gateway-api/src/admin/settings.rs +++ b/backend/crates/gateway-api/src/admin/settings.rs @@ -36,6 +36,7 @@ pub type ProviderRequestProfileUpdates = #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RuntimeSettingsFields { + pub codex_privacy_policy: gateway_core::settings::privacy::CodexPrivacyPolicy, pub request_location_enabled: bool, pub request_location: gateway_core::account::RequestLocation, pub refresh_margin_seconds: u64, @@ -283,6 +284,10 @@ where "/api/admin/settings/client-profiles/{provider}/preview", post(preview_client_profile::), ) + .route( + "/api/admin/settings/privacy/preview", + post(preview_privacy::), + ) .route("/api/admin/settings/update", post(update_settings::)) .route( "/api/admin/settings/client-downloads/codex-desktop/windows", @@ -719,6 +724,7 @@ impl RuntimeSettingsFields { self, ) -> Result { let values = gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: self.codex_privacy_policy, request_location_enabled: self.request_location_enabled, request_location: self .request_location @@ -767,6 +773,7 @@ impl RuntimeSettingsFields { impl From for RuntimeSettingsFields { fn from(settings: gateway_admin::model::settings::RuntimeSettingsValues) -> Self { crate::admin::settings::RuntimeSettingsFields { + codex_privacy_policy: settings.codex_privacy_policy, request_location_enabled: settings.request_location_enabled, request_location: settings.request_location, refresh_margin_seconds: settings.refresh_margin_seconds, @@ -837,3 +844,23 @@ fn settings_validation_error(field: &'static str) -> WireValidationError { _ => "settings", }) } + +async fn preview_privacy( + State(state): State, + _auth: AdminAuth, + AdminJson(request): AdminJson, +) -> Result +where + S: SessionState + Send + Sync, +{ + let result = state + .admin_services() + .settings() + .preview_privacy_policy(request) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(result), + )) +} diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index e451ecaae..5986fbca4 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -600,6 +600,7 @@ impl SettingsStore for MemorySettingsStore { rotation_strategy: command.rotation_strategy, updated_at: Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: command.values.codex_privacy_policy, request_location_enabled: command.values.request_location_enabled, request_location: command.values.request_location, refresh_margin_seconds: command.values.refresh_margin_seconds, @@ -1667,6 +1668,7 @@ fn test_runtime_settings() -> RuntimeSettings { rotation_strategy: RotationStrategy::Smart, updated_at: Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 3_600, diff --git a/backend/crates/gateway-api/tests/admin/settings.rs b/backend/crates/gateway-api/tests/admin/settings.rs index b14d8d93a..80cc6d41c 100644 --- a/backend/crates/gateway-api/tests/admin/settings.rs +++ b/backend/crates/gateway-api/tests/admin/settings.rs @@ -40,6 +40,7 @@ async fn response_json(response: axum::response::Response) -> Value { fn update_body() -> Value { json!({ "configRevision": 7, + "codexPrivacyPolicy": {"enabled":false,"onError":"skip_rule","rules":[]}, "requestLocationEnabled": false, "requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"}, "modelMappings": { @@ -78,6 +79,63 @@ fn update_body() -> Value { }) } +#[tokio::test] +async fn privacy_save_requires_provider_validation_and_preview_requires_admin_auth() { + let fixture = AdminTestFixture::new().await; + fixture.auth.insert_session("valid-session"); + let mut body = update_body(); + let policy = json!({"enabled":true,"onError":"reject_request","rules":[{ + "id":"auth","name":"自定义认证头","enabled":true,"scope":"request_header", + "selector":"authorization","action":"remove_field","pattern":null, + "replacement":"","value":null,"replaceAll":true,"caseInsensitive":false,"multiLine":false + }]}); + body["codexPrivacyPolicy"] = policy.clone(); + let response = app(fixture.state()) + .oneshot(request( + Method::POST, + "/api/admin/settings/update", + Some(body), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = app(fixture.state()) + .oneshot(request(Method::GET, "/api/admin/settings", None)) + .await + .unwrap(); + assert_eq!( + response_json(response).await["data"]["codexPrivacyPolicy"], + json!({"enabled":false,"onError":"skip_rule","rules":[]}) + ); + let response = app(fixture.state()) + .oneshot( + Request::builder() + .method(Method::POST) + .uri("/api/admin/settings/privacy/preview") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from( + json!({"policy":policy,"body":{},"headers":{},"turnMetadata":null}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[test] +fn privacy_policy_is_required_and_rejects_unknown_contract_fields() { + let mut missing = update_body(); + missing + .as_object_mut() + .unwrap() + .remove("codexPrivacyPolicy"); + assert!(serde_json::from_value::(missing).is_err()); + let mut unknown = update_body(); + unknown["codexPrivacyPolicy"]["protectedFields"] = json!([]); + assert!(serde_json::from_value::(unknown).is_err()); +} + #[tokio::test] async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_intact() { let fixture = AdminTestFixture::new().await; @@ -223,6 +281,7 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() { .single() .expect("timestamp"), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: 1800, @@ -268,6 +327,7 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() { "providerRequestProfiles": {}, "openaiClientProfile": null, "xaiClientProfile": null, + "codexPrivacyPolicy": {"enabled":false,"onError":"skip_rule","rules":[]}, "requestLocationEnabled": false, "requestLocation": {"country":"US", "region":"Ohio", "city":"Piketon", "timezone":"America/New_York"}, "modelMappings": { @@ -348,6 +408,7 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() { .expect("fixture rotation strategy"), updated_at: chrono::Utc::now(), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds: request.values.refresh_margin_seconds, diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index 1c56882b0..5f8c8b64e 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -1045,6 +1045,7 @@ where .with_fast_mode(self.plan.fast_mode()) .with_pricing(self.plan.pricing()) .with_request_location(self.plan.request_location().cloned()) + .with_privacy(self.plan.privacy()) .with_concurrency_wait_budget(self.concurrency_wait_budget.clone()) .with_connection_budget(self.connection_budget.clone()) .with_timing_started_at(self.observation.timing_started_at) diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index 64636af8f..e11608186 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -287,6 +287,7 @@ pub struct RequestAttemptContext { request_profile: Option, fast_mode: FastMode, request_location: Option, + privacy: Option>, request_id: ModelRequestId, client_api_key_ref: ClientApiKeyId, timing_started_at: Instant, @@ -350,6 +351,15 @@ impl RequestAttemptContext { self } + #[must_use] + pub fn with_privacy( + mut self, + privacy: Option>, + ) -> Self { + self.privacy = privacy; + self + } + #[must_use] pub fn with_request_location( mut self, @@ -369,6 +379,7 @@ impl RequestAttemptContext { pricing: Arc::default(), fast_mode: FastMode::Default, request_location: None, + privacy: None, timing_started_at: Instant::now(), trace: crate::diagnostics::TraceContext::default(), concurrency_wait_budget: crate::concurrency::ConcurrencyWaitBudget::default(), @@ -569,6 +580,11 @@ impl AttemptContext { self.request.fast_mode } + #[must_use] + pub fn privacy(&self) -> Option> { + self.request.privacy.clone() + } + #[must_use] pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> { self.request.request_location.as_ref() diff --git a/backend/crates/gateway-core/src/engine/provider.rs b/backend/crates/gateway-core/src/engine/provider.rs index 3ce482cc1..a72371357 100644 --- a/backend/crates/gateway-core/src/engine/provider.rs +++ b/backend/crates/gateway-core/src/engine/provider.rs @@ -965,6 +965,20 @@ impl fmt::Debug for ProviderRequest { /// 新的 attempt 再次调用 #[async_trait] pub trait Provider: Send + Sync { + /// Provider 解释自己的隐私规则，Core 仅冻结编译结果 + fn compile_privacy_policy( + &self, + _policy: &crate::settings::privacy::CodexPrivacyPolicy, + ) -> Result< + Arc, + crate::settings::privacy::PrivacyError, + > { + Err(crate::settings::privacy::PrivacyError { + rule_index: 0, + reason: "Provider 不支持隐私策略", + }) + } + /// 从已冻结的配置解析请求身份；只读取本地发布资料，不执行网络请求 fn resolve_request_profile( &self, @@ -1197,3 +1211,31 @@ impl ProviderCatalogPort for ProviderRegistry { }) } } + +impl std::fmt::Debug for ProviderRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProviderRegistry") + .field("providers", &self.providers.keys()) + .finish() + } +} + +impl crate::settings::privacy::PrivacyPolicyCompiler for ProviderRegistry { + fn compile( + &self, + policy: &crate::settings::privacy::CodexPrivacyPolicy, + ) -> Result< + Arc, + crate::settings::privacy::PrivacyError, + > { + self.providers + .iter() + .find(|(kind, _)| kind.as_str() == "openai") + .ok_or(crate::settings::privacy::PrivacyError { + rule_index: 0, + reason: "OpenAI Provider 不可用", + })? + .1 + .compile_privacy_policy(policy) + } +} diff --git a/backend/crates/gateway-core/src/lib.rs b/backend/crates/gateway-core/src/lib.rs index 7b206691f..fb62c1a4e 100644 --- a/backend/crates/gateway-core/src/lib.rs +++ b/backend/crates/gateway-core/src/lib.rs @@ -472,7 +472,8 @@ pub fn prepare_control_plane( providers: ProviderRegistry, extensions: Option>, ) -> CoreControlPlaneStartup { - let mut compiler = RuntimeSnapshotCompiler::new(ports.snapshots, Arc::new(providers)); + let mut compiler = RuntimeSnapshotCompiler::new(ports.snapshots, Arc::new(providers.clone())) + .with_privacy_compiler(Arc::new(providers)); if let Some(extensions) = extensions { compiler = compiler.with_extensions(extensions); } diff --git a/backend/crates/gateway-core/src/routing/mod.rs b/backend/crates/gateway-core/src/routing/mod.rs index 532b212b5..d81dbcdb5 100644 --- a/backend/crates/gateway-core/src/routing/mod.rs +++ b/backend/crates/gateway-core/src/routing/mod.rs @@ -563,6 +563,7 @@ impl ProviderCandidate { pub struct RoutingPlan { pricing: Arc, request_location: Option, + privacy: Option>, config_revision: ConfigRevision, account_selection_policy: AccountSelectionPolicy, operation: OperationKind, @@ -582,6 +583,12 @@ impl RoutingPlan { self.account_scope.fast_mode() } + /// 本次请求冻结的已编译规则，重试不读取新的全局策略 + #[must_use] + pub fn privacy(&self) -> Option> { + self.privacy.clone() + } + /// 本次请求冻结的全局位置，重试时沿用同一份配置 #[must_use] pub const fn request_location(&self) -> Option<&crate::account::RequestLocation> { diff --git a/backend/crates/gateway-core/src/routing/snapshot.rs b/backend/crates/gateway-core/src/routing/snapshot.rs index c5dd3fe50..daa89c320 100644 --- a/backend/crates/gateway-core/src/routing/snapshot.rs +++ b/backend/crates/gateway-core/src/routing/snapshot.rs @@ -219,6 +219,7 @@ pub enum RuntimeSnapshotCompileError { #[derive(Clone)] pub struct RuntimeSnapshotCompiler { store: Arc, + privacy_compiler: Option>, catalogs: Arc, extensions: Option>, } @@ -231,6 +232,7 @@ impl RuntimeSnapshotCompiler { ) -> Self { Self { store, + privacy_compiler: None, catalogs, extensions: None, } @@ -245,6 +247,15 @@ impl RuntimeSnapshotCompiler { self } + #[must_use] + pub fn with_privacy_compiler( + mut self, + compiler: Arc, + ) -> Self { + self.privacy_compiler = Some(compiler); + self + } + pub(crate) fn store(&self) -> Arc { Arc::clone(&self.store) } @@ -314,8 +325,14 @@ impl RuntimeSnapshotCompiler { let cached = previous.filter(|previous| { previous.extensions().map(|set| set.id()) == extensions.as_ref().map(|set| set.id()) }); - let snapshot = - compile_runtime_snapshot(facts, catalogs.as_ref(), provider_kinds, cached).await?; + let snapshot = compile_runtime_snapshot( + facts, + catalogs.as_ref(), + provider_kinds, + cached, + self.privacy_compiler.clone(), + ) + .await?; let observed_generations = catalogs.catalog_generations(); if catalog_generations == observed_generations { if extensions.is_some() @@ -348,6 +365,7 @@ async fn compile_runtime_snapshot( catalogs: &dyn ProviderCatalogPort, provider_kinds: Vec, previous: Option<&RuntimeSnapshot>, + privacy_compiler: Option>, ) -> Result { // 只有成功取得的完整目录能证明模型缺项；发现型目录和查询失败均交由上游验证 let mut provider_models = Vec::new(); @@ -504,12 +522,13 @@ async fn compile_runtime_snapshot( )); } - RuntimeSnapshot::new( + RuntimeSnapshot::new_with_privacy_compiler( facts.config_revision, facts.settings, provider_kinds, provider_models, client_policies, + privacy_compiler, ) .map_err(|_| RuntimeSnapshotCompileError::InvalidData) .map(|snapshot| { @@ -524,6 +543,7 @@ async fn compile_runtime_snapshot( #[derive(Debug, Clone)] pub struct RuntimeSnapshot { settings: Arc, + privacy_compiler: Option>, extensions: Option, revision: ConfigRevision, providers: Arc>, @@ -559,8 +579,12 @@ impl RuntimeSnapshot { &self, values: &SettingsValues, ) -> Result { - let settings = CompiledSettings::new(values.clone()) - .map_err(|_| RuntimeSnapshotCompileError::InvalidData)?; + let settings = CompiledSettings::new( + values.clone(), + self.privacy_compiler.as_deref(), + Some(&self.settings), + ) + .map_err(|_| RuntimeSnapshotCompileError::InvalidData)?; Ok(Self { settings: Arc::new(settings), ..self.clone() @@ -649,6 +673,24 @@ impl RuntimeSnapshot { providers: Vec, provider_models: Vec, client_policies: Vec, + ) -> Result { + Self::new_with_privacy_compiler( + revision, + values, + providers, + provider_models, + client_policies, + None, + ) + } + + pub fn new_with_privacy_compiler( + revision: ConfigRevision, + values: SettingsValues, + providers: Vec, + provider_models: Vec, + client_policies: Vec, + privacy_compiler: Option>, ) -> Result { let mut provider_set = BTreeSet::new(); for provider in providers { @@ -709,9 +751,11 @@ impl RuntimeSnapshot { } client_policy_map.retain(|_, policy| policy.enabled()); - let settings = CompiledSettings::new(values).map_err(|_| RoutingError::InvalidSettings)?; + let settings = CompiledSettings::new(values, privacy_compiler.as_deref(), None) + .map_err(|_| RoutingError::InvalidSettings)?; Ok(Self { settings: Arc::new(settings), + privacy_compiler, extensions: None, revision, providers: Arc::new(provider_set), @@ -1143,6 +1187,7 @@ impl RuntimeSnapshot { config_revision: self.revision, pricing: Arc::clone(&self.settings.values.pricing), request_location: self.settings.request_location.clone(), + privacy: self.settings.privacy.clone(), account_selection_policy: self.settings.account_selection_policy, operation: operation.kind(), max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS) @@ -1223,6 +1268,7 @@ impl RuntimeSnapshot { config_revision: self.revision, pricing: Arc::clone(&self.settings.values.pricing), request_location: self.settings.request_location.clone(), + privacy: self.settings.privacy.clone(), account_selection_policy: self.settings.account_selection_policy, operation: operation.kind(), max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS) diff --git a/backend/crates/gateway-core/src/settings/compiled.rs b/backend/crates/gateway-core/src/settings/compiled.rs index 92be6a5de..2721758a4 100644 --- a/backend/crates/gateway-core/src/settings/compiled.rs +++ b/backend/crates/gateway-core/src/settings/compiled.rs @@ -13,6 +13,7 @@ use crate::{ #[derive(Debug, Clone)] pub(crate) struct CompiledSettings { pub(crate) values: SettingsValues, + pub(crate) privacy: Option>, pub(crate) account_selection_policy: AccountSelectionPolicy, pub(crate) client_queue_policy: ConcurrencyQueuePolicy, pub(crate) responses_max_decompressed_body_bytes: std::num::NonZeroUsize, @@ -21,7 +22,34 @@ pub(crate) struct CompiledSettings { } impl CompiledSettings { - pub(crate) fn new(settings: SettingsValues) -> Result { + pub(crate) fn new( + settings: SettingsValues, + compiler: Option<&dyn super::privacy::PrivacyPolicyCompiler>, + previous: Option<&Self>, + ) -> Result { + let privacy = if let Some(previous) = previous.filter(|previous| { + previous.values.codex_privacy_policy == settings.codex_privacy_policy + }) { + previous.privacy.clone() + } else if settings.codex_privacy_policy == super::privacy::CodexPrivacyPolicy::default() { + None + } else { + Some( + compiler + .ok_or(InvalidSettings)? + .compile(&settings.codex_privacy_policy) + .map_err(|_| InvalidSettings)?, + ) + }; + // 关闭或没有启用规则时不进入出站管线，保留原始 JSON 字节和重复键语义 + let privacy = privacy.filter(|_| { + settings.codex_privacy_policy.enabled + && settings + .codex_privacy_policy + .rules + .iter() + .any(|rule| rule.enabled) + }); super::validate_request_limits( settings.max_waiting_per_key, settings.max_waiting_per_account, @@ -79,6 +107,7 @@ impl CompiledSettings { min_codex_client_versions, request_location, values: settings, + privacy, }) } } diff --git a/backend/crates/gateway-core/src/settings/mod.rs b/backend/crates/gateway-core/src/settings/mod.rs index 10adbfb08..9efc4fdd7 100644 --- a/backend/crates/gateway-core/src/settings/mod.rs +++ b/backend/crates/gateway-core/src/settings/mod.rs @@ -1,6 +1,7 @@ //! 运行设置的纯值、校验与编译结果 pub(crate) mod compiled; +pub mod privacy; mod values; pub use values::{ SettingsValues, client_min_versions, response_body_limit, validate_request_limits, diff --git a/backend/crates/gateway-core/src/settings/privacy.rs b/backend/crates/gateway-core/src/settings/privacy.rs new file mode 100644 index 000000000..8dbcb717d --- /dev/null +++ b/backend/crates/gateway-core/src/settings/privacy.rs @@ -0,0 +1,134 @@ +//! 隐私策略配置、编译端口与无敏感值的执行结果，协议解析由 Provider 拥有 + +use std::{fmt, sync::Arc}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CodexPrivacyPolicy { + pub enabled: bool, + pub on_error: PrivacyFailureMode, + pub rules: Vec, +} + +impl fmt::Debug for CodexPrivacyPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CodexPrivacyPolicy") + .field("enabled", &self.enabled) + .field("on_error", &self.on_error) + .field("rule_count", &self.rules.len()) + .finish() + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyFailureMode { + #[default] + SkipRule, + RejectRequest, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyScope { + TurnMetadata, + DesktopGitContext, + EnvironmentText, + RequestBody, + RequestHeader, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyAction { + RegexReplace, + SetValue, + RenameKey, + RemoveField, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PrivacyRule { + pub id: String, + pub name: String, + pub enabled: bool, + pub scope: PrivacyScope, + pub selector: String, + pub action: PrivacyAction, + /// 删除动作的 null 表示无条件删除，字符串模式只匹配目标字符串值 + pub pattern: Option, + pub replacement: String, + pub value: Value, + pub replace_all: bool, + pub case_insensitive: bool, + pub multi_line: bool, +} + +impl fmt::Debug for PrivacyRule { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PrivacyRule") + .field("scope", &self.scope) + .field("action", &self.action) + .field("enabled", &self.enabled) + .finish_non_exhaustive() + } +} + +/// 编译和执行失败仅携带规则序号与静态原因，不回显表达式或请求内容 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error, Serialize)] +#[serde(rename_all = "camelCase")] +#[error("privacy rule {rule_index}: {reason}")] +pub struct PrivacyError { + pub rule_index: usize, + pub reason: &'static str, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyRuleOutcome { + pub rule_id: String, + pub matches: usize, + pub status: String, + pub reason: Option, +} + +/// 已编译策略随快照冻结，调用方提供独立的本次出站副本 +pub trait CompiledPrivacyPolicy: fmt::Debug + Send + Sync { + fn apply( + &self, + body: &mut Value, + headers: &mut http::HeaderMap, + turn_metadata: &mut Option, + cancelled: &dyn Fn() -> bool, + ) -> Result, PrivacyError>; +} + +pub trait PrivacyPolicyCompiler: fmt::Debug + Send + Sync { + fn compile( + &self, + policy: &CodexPrivacyPolicy, + ) -> Result, PrivacyError>; +} + +/// 预览只接受管理员主动提供的受限样本，不读取实际请求记录 +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PrivacyPreviewRequest { + pub policy: CodexPrivacyPolicy, + pub body: Value, + pub headers: std::collections::BTreeMap>, + pub turn_metadata: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyPreviewResult { + pub body: Value, + pub headers: std::collections::BTreeMap>, + pub turn_metadata: Option, + pub outcomes: Vec, +} diff --git a/backend/crates/gateway-core/src/settings/values.rs b/backend/crates/gateway-core/src/settings/values.rs index a75c075c9..90a840f5b 100644 --- a/backend/crates/gateway-core/src/settings/values.rs +++ b/backend/crates/gateway-core/src/settings/values.rs @@ -10,6 +10,7 @@ use crate::identity::ProviderKind; pub struct SettingsValues { pub(crate) pricing: Arc, pub(crate) request_profiles: Arc>, + pub(crate) codex_privacy_policy: super::privacy::CodexPrivacyPolicy, pub(crate) request_location_enabled: bool, pub(crate) request_location: crate::account::RequestLocation, pub(crate) max_concurrent_per_account: u32, @@ -30,6 +31,12 @@ pub struct SettingsValues { } impl SettingsValues { + #[must_use] + pub fn with_codex_privacy_policy(mut self, policy: super::privacy::CodexPrivacyPolicy) -> Self { + self.codex_privacy_policy = policy; + self + } + #[must_use] pub const fn with_openai_session_affinity_ttl_hours(mut self, hours: u32) -> Self { self.openai_session_affinity_ttl_hours = hours; @@ -131,6 +138,7 @@ impl SettingsValues { Self { request_profiles: Arc::default(), pricing: Arc::default(), + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: crate::account::RequestLocation::default(), max_concurrent_per_account, diff --git a/backend/crates/gateway-core/tests/routing/mod.rs b/backend/crates/gateway-core/tests/routing/mod.rs index 92d21416c..86be922d4 100644 --- a/backend/crates/gateway-core/tests/routing/mod.rs +++ b/backend/crates/gateway-core/tests/routing/mod.rs @@ -18,6 +18,7 @@ use gateway_core::routing::{ UpstreamModelId, }; +mod privacy; mod request_settings; mod snapshot; diff --git a/backend/crates/gateway-core/tests/routing/privacy.rs b/backend/crates/gateway-core/tests/routing/privacy.rs new file mode 100644 index 000000000..51e9cd1f5 --- /dev/null +++ b/backend/crates/gateway-core/tests/routing/privacy.rs @@ -0,0 +1,125 @@ +//! 隐私策略在快照与请求计划中的冻结、复用及关闭行为 + +use super::*; +use gateway_core::settings::privacy::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug, Default)] +struct Compiler(AtomicUsize); + +#[derive(Debug)] +struct Compiled(serde_json::Value); + +impl PrivacyPolicyCompiler for Compiler { + fn compile( + &self, + policy: &CodexPrivacyPolicy, + ) -> Result, PrivacyError> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(Compiled(policy.rules[0].value.clone()))) + } +} + +impl CompiledPrivacyPolicy for Compiled { + fn apply( + &self, + body: &mut serde_json::Value, + _: &mut http::HeaderMap, + _: &mut Option, + _: &dyn Fn() -> bool, + ) -> Result, PrivacyError> { + *body = self.0.clone(); + Ok(Vec::new()) + } +} + +fn policy(value: &str) -> CodexPrivacyPolicy { + CodexPrivacyPolicy { + enabled: true, + on_error: PrivacyFailureMode::RejectRequest, + rules: vec![PrivacyRule { + id: "rule".into(), + name: "规则".into(), + enabled: true, + scope: PrivacyScope::RequestBody, + selector: "$.metadata".into(), + action: PrivacyAction::SetValue, + pattern: None, + replacement: String::new(), + value: serde_json::json!(value), + replace_all: false, + case_insensitive: false, + multi_line: false, + }], + } +} + +fn compiled(snapshot: &RuntimeSnapshot) -> Option> { + snapshot + .plan_provider_endpoint( + &ProviderKind::new("openai").unwrap(), + None, + &image_operation(), + snapshot.all_account_scope(), + &RoutingContext::default(), + ) + .unwrap() + .privacy() +} + +#[test] +fn privacy_is_frozen_and_unrelated_request_settings_reuse_compiled_rules() { + let compiler = Arc::new(Compiler::default()); + let snapshot = RuntimeSnapshot::new_with_privacy_compiler( + ConfigRevision::new(1).unwrap(), + settings().with_codex_privacy_policy(policy("first")), + vec![ProviderKind::new("openai").unwrap()], + vec![], + vec![], + Some(compiler.clone()), + ) + .unwrap() + .with_account_directory(account_directory()); + let original = compiled(&snapshot).unwrap(); + let limits = snapshot + .with_settings(&snapshot.settings().clone().with_max_account_rotations(5)) + .unwrap(); + assert!(Arc::ptr_eq(&original, &compiled(&limits).unwrap())); + assert_eq!(compiler.0.load(Ordering::SeqCst), 1); + let changed = snapshot + .with_settings( + &snapshot + .settings() + .clone() + .with_codex_privacy_policy(policy("second")), + ) + .unwrap(); + let mut body = serde_json::Value::Null; + original + .apply(&mut body, &mut http::HeaderMap::new(), &mut None, &|| false) + .unwrap(); + assert_eq!(body, "first"); + compiled(&changed) + .unwrap() + .apply(&mut body, &mut http::HeaderMap::new(), &mut None, &|| false) + .unwrap(); + assert_eq!(body, "second"); + assert_eq!(compiler.0.load(Ordering::SeqCst), 2); + for disable_rule in [false, true] { + let mut policy = policy("unused"); + if disable_rule { + policy.rules[0].enabled = false; + } else { + policy.enabled = false; + } + let disabled = snapshot + .with_settings( + &snapshot + .settings() + .clone() + .with_codex_privacy_policy(policy), + ) + .unwrap(); + assert!(compiled(&disabled).is_none()); + } +} diff --git a/backend/crates/gateway-plugin/sdk/src/call/services/settings.rs b/backend/crates/gateway-plugin/sdk/src/call/services/settings.rs index bab9cdc92..600fb3c35 100644 --- a/backend/crates/gateway-plugin/sdk/src/call/services/settings.rs +++ b/backend/crates/gateway-plugin/sdk/src/call/services/settings.rs @@ -16,8 +16,80 @@ pub type RotationStrategy = String; pub type AccountAffinity = String; pub type PricingOverrides = BTreeMap>; #[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CodexPrivacyPolicy { + pub enabled: bool, + pub on_error: PrivacyFailureMode, + pub rules: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyFailureMode { + SkipRule, + RejectRequest, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyScope { + TurnMetadata, + DesktopGitContext, + EnvironmentText, + RequestBody, + RequestHeader, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyAction { + RegexReplace, + SetValue, + RenameKey, + RemoveField, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PrivacyRule { + pub id: String, + pub name: String, + pub enabled: bool, + pub scope: PrivacyScope, + pub selector: String, + pub action: PrivacyAction, + /// 删除动作的 null 表示无条件删除，字符串模式只匹配目标字符串值 + pub pattern: Option, + pub replacement: String, + pub value: serde_json::Value, + pub replace_all: bool, + pub case_insensitive: bool, + pub multi_line: bool, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyRuleOutcome { + pub rule_id: String, + pub matches: usize, + pub status: String, + pub reason: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PrivacyPreviewRequest { + pub policy: CodexPrivacyPolicy, + pub body: serde_json::Value, + pub headers: BTreeMap>, + pub turn_metadata: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyPreviewResult { + pub body: serde_json::Value, + pub headers: BTreeMap>, + pub turn_metadata: Option, + pub outcomes: Vec, +} +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct RuntimeSettings { + pub codex_privacy_policy: CodexPrivacyPolicy, pub request_location_enabled: bool, pub request_location: RequestLocation, pub refresh_margin_seconds: u64, @@ -57,6 +129,7 @@ pub struct RuntimeSettings { #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ReplaceRuntimeSettings { + pub codex_privacy_policy: CodexPrivacyPolicy, pub request_location_enabled: bool, pub request_location: RequestLocation, pub refresh_margin_seconds: u64, @@ -190,6 +263,12 @@ impl Operation for Load { type Input = (); type Output = RuntimeSettings; } +pub struct PreviewPrivacyPolicy; +impl Operation for PreviewPrivacyPolicy { + const NAME: &'static str = "settings.preview_privacy_policy"; + type Input = PrivacyPreviewRequest; + type Output = PrivacyPreviewResult; +} pub struct Replace; impl Operation for Replace { const NAME: &'static str = "settings.replace"; @@ -253,6 +332,7 @@ impl Operation for PreviewClientProfile { impl From for ReplaceRuntimeSettings { fn from(settings: RuntimeSettings) -> Self { Self { + codex_privacy_policy: settings.codex_privacy_policy, request_location_enabled: settings.request_location_enabled, request_location: settings.request_location, refresh_margin_seconds: settings.refresh_margin_seconds, diff --git a/backend/crates/gateway-store/src/admin_adapter.rs b/backend/crates/gateway-store/src/admin_adapter.rs index 140127fed..0d4322a06 100644 --- a/backend/crates/gateway-store/src/admin_adapter.rs +++ b/backend/crates/gateway-store/src/admin_adapter.rs @@ -96,6 +96,7 @@ impl SettingsStore for AdminSettingsStoreAdapter { "provider_request_profiles_json".to_owned(), "request_location_enabled".to_owned(), "request_location_json".to_owned(), + "codex_privacy_policy_json".to_owned(), "model_mappings_json".to_owned(), "refresh_margin_seconds".to_owned(), "refresh_concurrency".to_owned(), diff --git a/backend/crates/gateway-store/src/postgres/runtime_settings.rs b/backend/crates/gateway-store/src/postgres/runtime_settings.rs index d4114d7b9..46ed94781 100644 --- a/backend/crates/gateway-store/src/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/src/postgres/runtime_settings.rs @@ -150,7 +150,7 @@ impl RuntimeSettingsRepository for PgRuntimeSettingsRepository { pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResult { let row = sqlx::query_as::<_, RuntimeSettingsRow>( - "select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, + "select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, codex_privacy_policy_json, refresh_concurrency, max_concurrent_per_account, request_interval_ms, rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days, audit_retention_days, min_codex_desktop_version, @@ -328,7 +328,7 @@ pub(crate) async fn load_runtime_settings_in_transaction( transaction: &mut Transaction<'_, Postgres>, ) -> StoreResult { let row = sqlx::query_as::<_, RuntimeSettingsRow>( - "select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, + "select provider_request_profiles_json, config_revision, admin_api_key, refresh_margin_seconds, request_location_json, request_location_enabled, codex_privacy_policy_json, refresh_concurrency, max_concurrent_per_account, request_interval_ms, rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days, audit_retention_days, min_codex_desktop_version, @@ -412,6 +412,7 @@ pub(crate) async fn update_runtime_settings_in_transaction( openai_account_affinity = $32, max_account_rotations = $33, openai_session_affinity_ttl_hours = $34, + codex_privacy_policy_json = $35, updated_at = now() where id = 1 returning config_revision", @@ -461,6 +462,7 @@ pub(crate) async fn update_runtime_settings_in_transaction( .bind(update.values.openai_account_affinity.as_str()) .bind(i64::from(update.values.max_account_rotations)) .bind(i64::from(update.values.openai_session_affinity_ttl_hours)) + .bind(sqlx::types::Json(&update.values.codex_privacy_policy)) .fetch_optional(&mut **transaction) .await .map_err(|source| postgres_unavailable("update runtime settings in transaction", source))? @@ -523,6 +525,8 @@ struct RuntimeSettingsRow { request_interval_ms: i64, smart_scheduling_json: sqlx::types::Json, rotation_strategy: String, + codex_privacy_policy_json: + sqlx::types::Json, request_location_enabled: bool, request_location_json: sqlx::types::Json, model_mappings_json: sqlx::types::Json>, @@ -579,6 +583,7 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult, + codex_privacy_policy_json: + sqlx::types::Json, request_location_enabled: bool, responses_max_decompressed_body_bytes: i64, provider_request_profiles_json: @@ -236,7 +238,7 @@ async fn load_settings( transaction: &mut Transaction<'_, Postgres>, ) -> StoreResult<(Revision, SettingsValues)> { let row = sqlx::query_as::<_, SnapshotSettingsRow>( - "select config_revision, max_concurrent_per_account, request_interval_ms, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1", + "select config_revision, max_concurrent_per_account, request_interval_ms, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1", ) .fetch_optional(&mut **transaction) .await @@ -281,6 +283,7 @@ async fn load_settings( ) }) .with_request_location(row.request_location_json.0, row.request_location_enabled) + .with_codex_privacy_policy(row.codex_privacy_policy_json.0) .with_concurrency_queues( to_u32(row.max_waiting_per_key)?, to_u32(row.max_waiting_per_account)?, diff --git a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs index 8f2adee0b..59995942a 100644 --- a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs @@ -18,6 +18,7 @@ pub(super) fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettin ("grok-latest".to_owned(), "grok-4.5".to_owned()), ]), values: gateway_admin::model::settings::RuntimeSettingsValues { + codex_privacy_policy: Default::default(), request_location_enabled: false, request_location: Default::default(), refresh_margin_seconds, @@ -58,6 +59,59 @@ fn runtime_settings_keep_account_rotation_global() { assert!(settings.validate().is_ok()); } +#[tokio::test] +async fn privacy_policy_upgrade_defaults_off_and_round_trips_in_snapshot() { + use gateway_core::settings::privacy::*; + use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository}; + let Some(database) = TestDatabase::create_through("privacy_policy", 25).await else { + return; + }; + super::TEST_MIGRATOR.run(&database.pool).await.unwrap(); + let repository = PgRuntimeSettingsRepository::new(database.pool.clone()); + let before = repository.load_runtime_settings().await.unwrap(); + assert_eq!( + before.values.codex_privacy_policy, + CodexPrivacyPolicy::default() + ); + let mut update = settings_with_margin(300); + update.values.codex_privacy_policy = CodexPrivacyPolicy { + enabled: true, + on_error: PrivacyFailureMode::RejectRequest, + rules: vec![PrivacyRule { + id: "remove-auth".into(), + name: "配置者控制".into(), + enabled: true, + scope: PrivacyScope::RequestHeader, + selector: "authorization".into(), + action: PrivacyAction::RemoveField, + pattern: None, + replacement: String::new(), + value: serde_json::Value::Null, + replace_all: true, + case_insensitive: false, + multi_line: false, + }], + }; + let expected = update.values.codex_privacy_policy.clone(); + repository.update_runtime_settings(update).await.unwrap(); + let after = repository.load_runtime_settings().await.unwrap(); + assert_eq!(after.values.codex_privacy_policy, expected); + assert!(after.config_revision > before.config_revision); + let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone()) + .load_runtime_snapshot() + .await + .unwrap(); + assert_eq!( + serde_json::to_value(snapshot.settings).unwrap()["codex_privacy_policy"], + serde_json::to_value(expected).unwrap() + ); + assert_eq!( + before.values.codex_privacy_policy, + CodexPrivacyPolicy::default() + ); + database.close().await; +} + #[tokio::test] async fn smart_settings_upgrade_preserves_selection_and_publishes_custom_config() { use gateway_core::account::SmartSchedulingConfig; @@ -1187,9 +1241,15 @@ async fn preferred_affinity_migration_defaults_to_strict_and_preserves_saved_mod .await .unwrap(); let repository = PgRuntimeSettingsRepository::new(database.pool.clone()); - let mut update = settings_with_margin(3600); - update.values.openai_account_affinity = mode; - repository.update_runtime_settings(update).await.unwrap(); + // 升级前只写当时已有列，现行 Repository 依赖完整迁移后的 schema + sqlx::query("update runtime_settings set openai_account_affinity = $1, config_revision = config_revision + 1, updated_at = now() where id = 1") + .bind(match mode { + AccountAffinity::Relaxed => "relaxed", + _ => "strict", + }) + .execute(&database.pool) + .await + .unwrap(); super::TEST_MIGRATOR.run(&database.pool).await.unwrap(); assert_eq!( repository diff --git a/backend/crates/providers/openai/Cargo.toml b/backend/crates/providers/openai/Cargo.toml index 5ae2d6ac6..aac599f26 100644 --- a/backend/crates/providers/openai/Cargo.toml +++ b/backend/crates/providers/openai/Cargo.toml @@ -35,6 +35,7 @@ hyper-util = { version = "0.1.20", features = ["client-legacy", "http1", "tokio" # vendored OpenSSL 固定 Linux 发布版的 TLS 画像，且不启用压缩特性。 native-tls = "=0.2.14" reqwest = { version = "=0.12.28", features = ["json", "stream", "cookies", "rustls-tls-native-roots", "socks"] } +regex = "1.13.0" roxmltree = "0.21.1" semver = "1.0.27" # 与官方 rustls 路径一致：aws-lc 支持 P-521，优先提供 X25519MLKEM768。 diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index 3291c843e..55b8226b2 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -164,6 +164,26 @@ impl OpenAiAdminProvider { #[async_trait] impl ProviderAdmin for OpenAiAdminProvider { + fn compile_privacy_policy( + &self, + policy: &gateway_core::settings::privacy::CodexPrivacyPolicy, + ) -> Result< + Arc, + gateway_core::settings::privacy::PrivacyError, + > { + crate::transport::privacy::compile(policy) + } + + fn preview_privacy_policy( + &self, + request: gateway_core::settings::privacy::PrivacyPreviewRequest, + ) -> Result< + gateway_core::settings::privacy::PrivacyPreviewResult, + gateway_core::settings::privacy::PrivacyError, + > { + crate::transport::privacy::preview(request) + } + fn account_capabilities( &self, _account_id: &ProviderAccountId, diff --git a/backend/crates/providers/openai/src/provider/execution.rs b/backend/crates/providers/openai/src/provider/execution.rs index 42414604a..2675e9777 100644 --- a/backend/crates/providers/openai/src/provider/execution.rs +++ b/backend/crates/providers/openai/src/provider/execution.rs @@ -343,7 +343,8 @@ impl CodexProvider { .for_account(lease.account()) .map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)? .with_responses_api_base_url(lease.authentication().responses_api_base_url()) - .with_middleware_headers(middleware_headers), + .with_middleware_headers(middleware_headers) + .with_privacy(context.privacy(), context.cancellation().clone()), response_origin: request.response_origin, endpoint_path: request.endpoint_path, body: request.body, diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index dcb622a3d..1b83c312c 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -890,6 +890,13 @@ pub(super) fn map_client_error( let connect_retry = !local_connection_capacity && matches!(&error, CodexClientError::Http(error) if transient_http_connect(error)); let mut failure = match &error { + CodexClientError::Privacy(_) => MappedProviderFailure::plain( + provider_error( + ProviderErrorKind::InvalidRequest, + UpstreamSendState::NotSent, + ) + .with_retry_prohibited(), + ), CodexClientError::ConnectionBudgetExhausted => MappedProviderFailure::plain( provider_error(ProviderErrorKind::Timeout, UpstreamSendState::NotSent) .with_connection_retry(gateway_core::engine::AttemptTransport::Fallback), @@ -1115,6 +1122,11 @@ fn client_diagnostic(error: &CodexClientError) -> Option { "invalid_sse", "OpenAI HTTP returned an invalid Responses event stream".to_owned(), ), + CodexClientError::Privacy(error) => ( + "prepare", + "privacy_policy_rejected", + format!("Privacy rule {}: {}", error.rule_index + 1, error.reason), + ), CodexClientError::InvalidHeaderName(_) => ( "prepare", "invalid_header_name", diff --git a/backend/crates/providers/openai/src/provider/mod.rs b/backend/crates/providers/openai/src/provider/mod.rs index ddde47be2..48e1a7154 100644 --- a/backend/crates/providers/openai/src/provider/mod.rs +++ b/backend/crates/providers/openai/src/provider/mod.rs @@ -376,6 +376,16 @@ impl fmt::Debug for CodexProvider { #[async_trait] impl Provider for CodexProvider { + fn compile_privacy_policy( + &self, + policy: &gateway_core::settings::privacy::CodexPrivacyPolicy, + ) -> Result< + Arc, + gateway_core::settings::privacy::PrivacyError, + > { + crate::transport::privacy::compile(policy) + } + fn resolve_request_profile( &self, configuration: &gateway_core::account::OpaqueProviderData, @@ -1016,7 +1026,8 @@ impl CodexProvider { .with_responses_api_base_url(lease.authentication().responses_api_base_url()) .with_connection_budget(context.connection_budget().clone()) .with_response_control(context.response_control().cloned()) - .with_middleware_headers(middleware_headers), + .with_middleware_headers(middleware_headers) + .with_privacy(context.privacy(), context.cancellation().clone()), response_origin: self.responses_url.clone(), request: upstream_request, upstream_model, diff --git a/backend/crates/providers/openai/src/transport/client.rs b/backend/crates/providers/openai/src/transport/client.rs index bac975b64..3aea968ee 100644 --- a/backend/crates/providers/openai/src/transport/client.rs +++ b/backend/crates/providers/openai/src/transport/client.rs @@ -195,6 +195,8 @@ impl fmt::Debug for CodexClientVisibleUpstreamResponse { /// Codex 上游 HTTP 客户端错误 #[derive(Error)] pub enum CodexClientError { + #[error("privacy policy rejected the request: {0}")] + Privacy(#[from] gateway_core::settings::privacy::PrivacyError), #[error("connection recovery budget exhausted")] ConnectionBudgetExhausted, /// Reqwest 传输失败 @@ -279,6 +281,7 @@ impl fmt::Debug for CodexClientError { Self::ConnectionBudgetExhausted => { formatter.write_str("CodexClientError::ConnectionBudgetExhausted") } + Self::Privacy(error) => std::fmt::Debug::fmt(error, formatter), Self::Http(_) => formatter.write_str("CodexClientError::Http([REDACTED])"), Self::HttpJson(_) => formatter.write_str("CodexClientError::HttpJson([REDACTED])"), Self::ErrorBodyRead { @@ -345,7 +348,8 @@ impl CodexClientError { Self::Upstream { transport, .. } | Self::ErrorBodyRead { transport, .. } => { Some(*transport) } - Self::ConnectionBudgetExhausted + Self::Privacy(_) + | Self::ConnectionBudgetExhausted | Self::CustomCa(_) | Self::InvalidHeaderName(_) | Self::InvalidHeaderValue(_) @@ -677,6 +681,10 @@ impl OpenAiUpstreamProtocol { /// Codex HTTP/SSE 上游客户端 #[derive(Clone)] pub struct CodexBackendClient { + pub(super) privacy: Option<( + Arc, + gateway_core::lifecycle::CancellationToken, + )>, pub(super) timezone: gateway_core::time::DeploymentTimeZone, pub(super) response_control: Option, pub(super) connection_budget: Option, diff --git a/backend/crates/providers/openai/src/transport/client_json.rs b/backend/crates/providers/openai/src/transport/client_json.rs index be5ca252a..5ae6a2c8f 100644 --- a/backend/crates/providers/openai/src/transport/client_json.rs +++ b/backend/crates/providers/openai/src/transport/client_json.rs @@ -16,7 +16,7 @@ use super::{ }; impl CodexBackendClient { - /// 向固定 Provider 端点发送一次原始 JSON；请求与成功响应正文均不经过 serde + /// 向固定 Provider 端点发送 JSON，仅启用隐私规则时解码请求正文 pub(crate) async fn post_raw_json( &self, endpoint_path: &'static str, @@ -49,6 +49,21 @@ impl CodexBackendClient { context.turn_metadata, ); self.append_middleware_headers(&mut headers)?; + let body = if self.privacy.is_some() { + let mut value: serde_json::Value = + serde_json::from_slice(&body).map_err(CodexClientError::RequestBodyEncode)?; + let original = value.clone(); + self.apply_privacy(&mut value, &mut headers)?; + if value == original { + body + } else { + bytes::Bytes::from( + serde_json::to_vec(&value).map_err(CodexClientError::RequestBodyEncode)?, + ) + } + } else { + body + }; let trace = context .trace diff --git a/backend/crates/providers/openai/src/transport/client_sse.rs b/backend/crates/providers/openai/src/transport/client_sse.rs index 92c8163e3..d54703608 100644 --- a/backend/crates/providers/openai/src/transport/client_sse.rs +++ b/backend/crates/providers/openai/src/transport/client_sse.rs @@ -66,6 +66,7 @@ impl CodexBackendClient { ) -> Self { let base_url = base_url.into().trim_end_matches('/').to_string(); Self { + privacy: None, timezone: Default::default(), connection_budget: None, response_control: None, @@ -113,7 +114,7 @@ impl CodexBackendClient { upstream_request: &CodexResponsesRequest, context: CodexRequestContext<'_>, ) -> CodexClientResult { - let headers = self.request_headers_for_http_response(upstream_request, context)?; + let mut headers = self.request_headers_for_http_response(upstream_request, context)?; let headers_started_at = Instant::now(); // OAuth 请求遵循 Codex 压缩合同；API Key 上游使用普通 JSON // Codex 上游只交付 SSE；即使下游请求 `stream: false`，也要上游流式执行， @@ -122,6 +123,11 @@ impl CodexBackendClient { // 直接透传给 Codex，否则上游会以 400 拒绝非流式请求 let mut upstream_body = upstream_request.body().clone(); upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true)); + let mut upstream_body = serde_json::Value::Object(upstream_body); + if self.protocol == OpenAiUpstreamProtocol::Codex { + headers.insert(CONTENT_ENCODING, HeaderValue::from_static("zstd")); + } + self.apply_privacy(&mut upstream_body, &mut headers)?; let body = serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?; let endpoint = endpoint_url(&self.base_url, self.protocol.responses_path()); @@ -141,9 +147,8 @@ impl CodexBackendClient { ); trace.capture("upstream.request.body", &body); let client = self.http_opening_client()?; - let mut outbound = client.post(endpoint).headers(headers); + let outbound = client.post(endpoint).headers(headers); let body = if self.protocol == OpenAiUpstreamProtocol::Codex { - outbound = outbound.header(CONTENT_ENCODING, HeaderValue::from_static("zstd")); zstd::stream::encode_all(std::io::Cursor::new(body), 3) .map_err(CodexClientError::RequestCompression)? } else { @@ -278,8 +283,18 @@ impl CodexBackendClient { }); } - let websocket_request = websocket_upstream_request(request); - let headers = self.request_headers_for_websocket_response(&websocket_request, context)?; + let mut websocket_request = websocket_upstream_request(request); + let mut headers = + self.request_headers_for_websocket_response(&websocket_request, context)?; + let mut body = serde_json::Value::Object(std::mem::take(websocket_request.body_mut())); + let original_headers = self.privacy.as_ref().map(|_| headers.clone()); + self.apply_privacy(&mut body, &mut headers)?; + *websocket_request.body_mut() = body.as_object().cloned().ok_or_else(|| { + CodexClientError::Privacy(gateway_core::settings::privacy::PrivacyError { + rule_index: 0, + reason: "WS 请求正文必须是对象", + }) + })?; let mut websocket_create = CodexWebSocketConnection::responses_create_request_for_path( &self.base_url, self.protocol.responses_path(), @@ -313,7 +328,11 @@ impl CodexBackendClient { tracing::warn!(error = %error, "Failed to write Codex WebSocket audit artifact"); } } - let connection_profile = websocket_connection_profile(&headers, &self.middleware_headers); + let connection_profile = websocket_connection_profile( + &headers, + &self.middleware_headers, + original_headers.as_ref(), + ); let pool_key = self.websocket_pool_key(request, context, pool_account_id, &connection_profile); let pool_log_context = pool_key.as_ref().map(WebSocketPoolLogContext::from_key); @@ -664,6 +683,7 @@ async fn read_model_catalog_body(response: ReqwestResponse) -> CodexClientResult fn websocket_connection_profile( headers: &HeaderMap, middleware_headers: &[gateway_core::engine::middleware::MiddlewareHeader], + original_headers: Option<&HeaderMap>, ) -> String { let mut profile = [ "originator", @@ -693,6 +713,38 @@ fn websocket_connection_profile( profile.push('\0'); profile.push_str(&hex::encode(digest.finalize())); } + if let Some(original) = original_headers { + use sha2::{Digest, Sha256}; + let names: std::collections::BTreeSet<_> = original + .keys() + .chain(headers.keys()) + .map(|name| name.as_str()) + .collect(); + let mut digest = Sha256::new(); + let mut changed = false; + for name in names { + if original + .get_all(name) + .iter() + .eq(headers.get_all(name).iter()) + { + continue; + } + changed = true; + digest.update(name.len().to_le_bytes()); + digest.update(name.as_bytes()); + digest.update(headers.get_all(name).iter().count().to_le_bytes()); + for value in headers.get_all(name) { + digest.update(value.len().to_le_bytes()); + digest.update(value.as_bytes()); + } + } + // 改写后的握手头必须参与池身份，避免复用仍携带旧隐私值的连接 + if changed { + profile.push('\0'); + profile.push_str(&hex::encode(digest.finalize())); + } + } profile } diff --git a/backend/crates/providers/openai/src/transport/mod.rs b/backend/crates/providers/openai/src/transport/mod.rs index cfe57ea97..c23657cc6 100644 --- a/backend/crates/providers/openai/src/transport/mod.rs +++ b/backend/crates/providers/openai/src/transport/mod.rs @@ -72,3 +72,5 @@ pub use self::{ CodexWebSocketPool, CodexWebSocketPoolConfig, CodexWebSocketPoolKey, WebSocketPoolDecision, }, }; + +pub mod privacy; diff --git a/backend/crates/providers/openai/src/transport/privacy/carrier.rs b/backend/crates/providers/openai/src/transport/privacy/carrier.rs new file mode 100644 index 000000000..c42d8021e --- /dev/null +++ b/backend/crates/providers/openai/src/transport/privacy/carrier.rs @@ -0,0 +1,288 @@ +//! 对独立 metadata 副本及已识别自动上下文进行解码、改写与原位写回 + +use gateway_core::settings::privacy::PrivacyScope; +use reqwest::header::{HeaderMap, HeaderValue}; +use serde_json::Value; + +use super::{Budget, rule::CompiledRule}; + +const METADATA_KEYS: &[&str] = &["turnMetadata", "turn_metadata", "x-codex-turn-metadata"]; + +pub(super) fn apply( + rule: &CompiledRule, + body: &mut Value, + headers: &mut HeaderMap, + turn_metadata: &mut Option, + budget: &mut Budget<'_>, +) -> Result { + match rule.rule.scope { + PrivacyScope::TurnMetadata => { + let mut count = 0; + if let Some(raw) = turn_metadata { + count += metadata(rule, raw, budget)?; + } + for (name, value) in headers.iter_mut() { + if name == "x-codex-turn-metadata" { + let mut raw = std::str::from_utf8(value.as_bytes()) + .map_err(|_| "metadata 请求头编码无效")? + .to_owned(); + count += metadata(rule, &mut raw, budget)?; + *value = HeaderValue::from_str(&raw).map_err(|_| "metadata 请求头编码无效")?; + } + } + count += metadata_fields(rule, body, budget)?; + if let Some(client) = body.get_mut("client_metadata") { + count += metadata_fields(rule, client, budget)?; + } + Ok(count) + } + PrivacyScope::DesktopGitContext | PrivacyScope::EnvironmentText => { + contexts(rule, body, budget) + } + PrivacyScope::RequestBody => rule.apply(body, budget), + PrivacyScope::RequestHeader => { + let name = reqwest::header::HeaderName::from_bytes(rule.rule.selector.as_bytes()) + .map_err(|_| "请求头名称无效")?; + let mut items = headers + .get_all(&name) + .iter() + .map(|value| { + std::str::from_utf8(value.as_bytes()) + .map(|s| Value::String(s.to_owned())) + .map_err(|_| "目标请求头不是文本") + }) + .collect::, _>>()?; + if items.is_empty() { + return Ok(0); + } + let mut count = 0; + for index in (0..items.len()).rev() { + // 复用同一目标执行器，删除根标记只在请求头承载点解释 + let removed = rule.apply_value(&mut items[index], budget, &mut count)?; + if removed { + items.remove(index); + } + } + if count == 0 { + return Ok(0); + } + headers.remove(&name); + for value in items { + let value = HeaderValue::from_str(value.as_str().ok_or("请求头只能保存字符串")?) + .map_err(|_| "替换结果不能作为请求头")?; + // WS opening 会忽略不能转为 ASCII 文本的头，预览应提前暴露这个失败 + value + .to_str() + .map_err(|_| "替换结果必须是 ASCII 请求头值")?; + headers.append(name.clone(), value); + } + Ok(count) + } + } +} + +fn metadata_fields( + rule: &CompiledRule, + object: &mut Value, + budget: &mut Budget<'_>, +) -> Result { + let mut count = 0; + for key in METADATA_KEYS { + if let Some(value) = object.get_mut(*key) { + match value { + Value::String(raw) => count += metadata(rule, raw, budget)?, + Value::Object(_) => count += rule.apply(value, budget)?, + _ => return Err("metadata 必须是 JSON 对象或编码后的字符串"), + } + } + } + Ok(count) +} + +fn metadata( + rule: &CompiledRule, + raw: &mut String, + budget: &mut Budget<'_>, +) -> Result { + budget.text(raw)?; + let mut value: Value = serde_json::from_str(raw).map_err(|_| "metadata JSON 无法解析")?; + if !value.is_object() { + return Err("metadata 必须是对象"); + } + let count = rule.apply(&mut value, budget)?; + if count > 0 { + let encoded = crate::transport::request::serialize_ascii_turn_metadata(&value) + .ok_or("metadata 无法编码")?; + if encoded.len() > super::MAX_VALUE_BYTES { + return Err("metadata 编码结果超出大小限制"); + } + *raw = encoded; + } + Ok(count) +} + +fn contexts( + rule: &CompiledRule, + body: &mut Value, + budget: &mut Budget<'_>, +) -> Result { + let Some(items) = body.get_mut("input").and_then(Value::as_array_mut) else { + return Ok(0); + }; + let mut count = 0; + for item in items { + budget.step()?; + let role = item + .get("role") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + let desktop = rule.rule.scope == PrivacyScope::DesktopGitContext; + if !(role == "user" || (!desktop && role == "developer")) { + continue; + } + let (kind, tag) = if desktop { + ( + "additional_content.codex_git_state", + "external_codex_git_state", + ) + } else if role == "developer" { + ( + "additional_content.codex_apps_client_time_context", + "codex_apps_client_time_context", + ) + } else { + ("environments.environment_context", "environment_context") + }; + let kinds = item + .pointer("/internal_chat_message_metadata_passthrough/content_item_kinds") + .cloned(); + let Some(parts) = item.get_mut("content").and_then(Value::as_array_mut) else { + continue; + }; + for (index, part) in parts.iter_mut().enumerate() { + if let Some(kinds) = &kinds + && kinds.get(index).and_then(Value::as_str) != Some(kind) + { + continue; + } + if part.get("type").and_then(Value::as_str) != Some("input_text") { + continue; + } + let Some(Value::String(text)) = part.get_mut("text") else { + continue; + }; + let full = text.trim().starts_with(&format!("<{tag}>")) + && text.trim().ends_with(&format!("")); + if !full && kinds.is_none() { + continue; + } + if !full { + return Err("已识别上下文不完整，可能已被截断"); + } + budget.text(text)?; + count += if desktop { + git(rule, text, budget)? + } else { + environment(rule, text, budget)? + }; + } + } + Ok(count) +} + +fn git( + rule: &CompiledRule, + text: &mut String, + budget: &mut Budget<'_>, +) -> Result { + let document = parse_xml(text)?; + budget.step()?; + let root = document.root_element(); + if root.tag_name().name() != "external_codex_git_state" { + return Err("Desktop Git 包裹不匹配"); + } + let nodes: Vec<_> = root + .descendants() + .filter(|node| { + node.has_tag_name("git") + && node + .parent() + .is_some_and(|p| p.has_tag_name("environment_context")) + }) + .collect(); + if nodes.len() != 1 { + return Err("Desktop Git 数据节点不明确"); + } + let git = nodes[0]; + if git.children().any(|node| !node.is_text()) { + return Err("Desktop Git 数据不是文本"); + } + let raw: String = git.children().filter_map(|node| node.text()).collect(); + let mut value: Value = serde_json::from_str(&raw).map_err(|_| "Desktop Git JSON 无法解析")?; + if !value.is_object() { + return Err("Desktop Git JSON 必须是对象"); + } + let count = rule.apply(&mut value, budget)?; + if count > 0 { + let range = git.range(); + let fragment = &text[range.clone()]; + let start = git + .first_child() + .ok_or("Desktop Git 数据为空")? + .range() + .start; + let end = range.start + fragment.rfind("', ">"); + if text.len() - (end - start) + escaped.len() > super::MAX_VALUE_BYTES { + return Err("Desktop Git 编码结果超出大小限制"); + } + text.replace_range(start..end, &escaped); + } + Ok(count) +} + +fn environment( + rule: &CompiledRule, + text: &mut String, + budget: &mut Budget<'_>, +) -> Result { + let before = xml_shape(text, budget)?; + let mut value = Value::String(text.clone()); + let count = rule.apply(&mut value, budget)?; + let result = value.as_str().ok_or("环境上下文必须是文本")?; + if xml_shape(result, budget)? != before { + return Err("替换不能改变环境上下文包裹结构"); + } + *text = result.to_owned(); + Ok(count) +} + +fn parse_xml(text: &str) -> Result, &'static str> { + roxmltree::Document::parse_with_options( + text, + roxmltree::ParsingOptions { + nodes_limit: 10_000, + ..Default::default() + }, + ) + .map_err(|_| "上下文 XML 无法解析或节点数量超限") +} + +fn xml_shape(text: &str, budget: &mut Budget<'_>) -> Result, &'static str> { + let document = parse_xml(text)?; + let mut shape = Vec::new(); + for node in document.descendants().filter(|node| node.is_element()) { + budget.step()?; + let depth = node.ancestors().take(65).count(); + if depth > 64 { + return Err("上下文 XML 嵌套超过 64 层"); + } + shape.push((node.tag_name().name().to_owned(), depth)); + } + Ok(shape) +} diff --git a/backend/crates/providers/openai/src/transport/privacy/mod.rs b/backend/crates/providers/openai/src/transport/privacy/mod.rs new file mode 100644 index 000000000..231b1ee9d --- /dev/null +++ b/backend/crates/providers/openai/src/transport/privacy/mod.rs @@ -0,0 +1,275 @@ +//! OpenAI 隐私规则的统一编译与执行，供设置预览和实际出站共享 + +mod carrier; +mod rule; +mod selector; + +use std::{ + collections::BTreeSet, + sync::Arc, + time::{Duration, Instant}, +}; + +use gateway_core::settings::privacy::{ + CodexPrivacyPolicy, CompiledPrivacyPolicy, PrivacyError, PrivacyFailureMode, + PrivacyPreviewRequest, PrivacyPreviewResult, PrivacyRuleOutcome, +}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use serde_json::Value; + +const MAX_VALUE_BYTES: usize = 1024 * 1024; +const MAX_REQUEST_BYTES: usize = 32 * 1024 * 1024; +const MAX_PREVIEW_BYTES: usize = 256 * 1024; + +struct Policy { + enabled: bool, + failure: PrivacyFailureMode, + rules: Vec, +} + +impl std::fmt::Debug for Policy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CompiledCodexPrivacyPolicy") + .field("rule_count", &self.rules.len()) + .finish() + } +} + +struct Budget<'a> { + started: Instant, + steps: usize, + text_bytes: usize, + cancelled: &'a dyn Fn() -> bool, +} + +impl Budget<'_> { + fn step(&mut self) -> Result<(), &'static str> { + self.steps += 1; + if (self.cancelled)() { + return Err("请求已取消或超时"); + } + if self.steps > 10_000 || self.started.elapsed() > Duration::from_millis(100) { + return Err("规则执行预算超限"); + } + Ok(()) + } + + fn text(&mut self, text: &str) -> Result<(), &'static str> { + self.step()?; + self.text_bytes = self.text_bytes.saturating_add(text.len()); + if text.len() > MAX_VALUE_BYTES || self.text_bytes > 16 * MAX_VALUE_BYTES { + return Err("目标文本超出大小限制"); + } + Ok(()) + } +} + +pub fn compile( + policy: &CodexPrivacyPolicy, +) -> Result, PrivacyError> { + if policy.rules.len() > 32 { + return Err(PrivacyError { + rule_index: 0, + reason: "最多配置 32 条规则", + }); + } + let mut ids = BTreeSet::new(); + let rules = policy + .rules + .iter() + .enumerate() + .map(|(rule_index, rule)| { + if !ids.insert(&rule.id) { + return Err(PrivacyError { + rule_index, + reason: "规则 ID 重复", + }); + } + rule::CompiledRule::compile(rule).map_err(|reason| PrivacyError { rule_index, reason }) + }) + .collect::, _>>()?; + Ok(Arc::new(Policy { + enabled: policy.enabled, + failure: policy.on_error, + rules, + })) +} + +impl CompiledPrivacyPolicy for Policy { + fn apply( + &self, + body: &mut Value, + headers: &mut HeaderMap, + turn_metadata: &mut Option, + cancelled: &dyn Fn() -> bool, + ) -> Result, PrivacyError> { + let mut outcomes = Vec::new(); + let mut budget = Budget { + started: Instant::now(), + steps: 0, + text_bytes: 0, + cancelled, + }; + for (rule_index, rule) in self.rules.iter().enumerate() { + let mut outcome = PrivacyRuleOutcome { + rule_id: rule.rule.id.clone(), + matches: 0, + status: "disabled".to_owned(), + reason: None, + }; + if self.enabled && rule.rule.enabled { + // 一条规则的所有承载副本同成同败，失败不会留下部分改写 + let result = (|| { + budget.step()?; + check_size(body, MAX_REQUEST_BYTES)?; + let mut next_body = body.clone(); + let mut next_headers = headers.clone(); + let mut next_metadata = turn_metadata.clone(); + let count = carrier::apply( + rule, + &mut next_body, + &mut next_headers, + &mut next_metadata, + &mut budget, + )?; + check_size(&next_body, MAX_REQUEST_BYTES)?; + budget.step()?; + *body = next_body; + *headers = next_headers; + *turn_metadata = next_metadata; + Ok(count) + })(); + match result { + Ok(count) => { + outcome.matches = count; + outcome.status = + if count == 0 { "unmatched" } else { "applied" }.to_owned(); + } + Err(reason) => { + if self.failure == PrivacyFailureMode::RejectRequest || cancelled() { + return Err(PrivacyError { rule_index, reason }); + } + outcome.status = "skipped".to_owned(); + outcome.reason = Some(reason.to_owned()); + } + } + } + outcomes.push(outcome); + } + Ok(outcomes) + } +} + +pub fn preview(request: PrivacyPreviewRequest) -> Result { + check_size(&request.body, MAX_PREVIEW_BYTES).map_err(|reason| PrivacyError { + rule_index: 0, + reason, + })?; + let size = request + .headers + .iter() + .map(|(key, values)| key.len() + values.iter().map(String::len).sum::()) + .sum::() + + request.turn_metadata.as_ref().map_or(0, String::len); + if size > MAX_PREVIEW_BYTES { + return Err(PrivacyError { + rule_index: 0, + reason: "预览样本最多 256 KiB", + }); + } + let mut headers = HeaderMap::new(); + for (key, values) in request.headers { + let name = HeaderName::from_bytes(key.as_bytes()).map_err(|_| PrivacyError { + rule_index: 0, + reason: "预览请求头名称无效", + })?; + for value in values { + headers.append( + name.clone(), + HeaderValue::from_str(&value).map_err(|_| PrivacyError { + rule_index: 0, + reason: "预览请求头值无效", + })?, + ); + } + } + let mut body = request.body; + let mut turn_metadata = request.turn_metadata; + let outcomes = + compile(&request.policy)?.apply(&mut body, &mut headers, &mut turn_metadata, &|| false)?; + let headers = headers + .keys() + .map(|key| { + ( + key.as_str().to_owned(), + headers + .get_all(key) + .iter() + .filter_map(|value| { + std::str::from_utf8(value.as_bytes()) + .ok() + .map(str::to_owned) + }) + .collect(), + ) + }) + .collect(); + Ok(PrivacyPreviewResult { + body, + headers, + turn_metadata, + outcomes, + }) +} + +fn check_size(value: &Value, limit: usize) -> Result<(), &'static str> { + struct Counter(usize); + impl std::io::Write for Counter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self + .0 + .checked_sub(bytes.len()) + .ok_or_else(|| std::io::Error::other("size limit"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + serde_json::to_writer(Counter(limit), value).map_err(|_| "请求或预览样本超出大小限制") +} + +impl super::client::CodexBackendClient { + pub(crate) fn with_privacy( + mut self, + policy: Option>, + cancellation: gateway_core::lifecycle::CancellationToken, + ) -> Self { + self.privacy = policy.map(|policy| (policy, cancellation)); + self + } + + /// 所有宿主头和 WS 投影完成后再执行，配置者可改写最终业务字段 + pub(super) fn apply_privacy( + &self, + body: &mut Value, + headers: &mut HeaderMap, + ) -> Result<(), super::client::CodexClientError> { + if let Some((policy, cancellation)) = &self.privacy { + let outcomes = + policy.apply(body, headers, &mut None, &|| cancellation.is_cancelled())?; + for (index, outcome) in outcomes + .iter() + .enumerate() + .filter(|(_, outcome)| outcome.status == "skipped") + { + tracing::warn!( + rule_index = index, + reason = outcome.reason, + "隐私规则未生效，按配置跳过" + ); + } + } + Ok(()) + } +} diff --git a/backend/crates/providers/openai/src/transport/privacy/rule.rs b/backend/crates/providers/openai/src/transport/privacy/rule.rs new file mode 100644 index 000000000..f3b5e70a2 --- /dev/null +++ b/backend/crates/providers/openai/src/transport/privacy/rule.rs @@ -0,0 +1,227 @@ +//! 单条规则的编译、捕获组校验和受预算约束的值修改 + +use std::collections::BTreeSet; + +use gateway_core::settings::privacy::{PrivacyAction, PrivacyRule, PrivacyScope}; +use regex::{Regex, RegexBuilder}; +use serde_json::Value; + +use super::{Budget, MAX_VALUE_BYTES, selector}; + +pub(super) struct CompiledRule { + pub(super) rule: PrivacyRule, + pub(super) path: Vec, + regex: Option, +} + +impl CompiledRule { + pub(super) fn compile(rule: &PrivacyRule) -> Result { + if rule.id.is_empty() || rule.id.len() > 64 || rule.name.is_empty() || rule.name.len() > 192 + { + return Err("规则 ID 或名称长度不合法"); + } + if rule.replacement.len() > 4096 + || serde_json::to_vec(&rule.value).map_or(true, |v| v.len() > 4096) + { + return Err("替换内容最多 4096 字节"); + } + let path = if rule.scope == PrivacyScope::RequestHeader { + Vec::new() + } else { + selector::parse(&rule.selector)? + }; + if rule.scope == PrivacyScope::RequestHeader { + reqwest::header::HeaderName::from_bytes(rule.selector.as_bytes()) + .map_err(|_| "请求头名称无效")?; + if rule.action == PrivacyAction::RenameKey { + return Err("请求头范围不支持对象键名替换"); + } + } + if rule.scope == PrivacyScope::EnvironmentText && !path.is_empty() { + return Err("环境文本使用 $ 选择整个文本值"); + } + let regex = rule + .pattern + .as_ref() + .map(|pattern| { + if pattern.len() > 4096 { + return Err("正则表达式最多 4096 字节"); + } + RegexBuilder::new(pattern) + .case_insensitive(rule.case_insensitive) + .multi_line(rule.multi_line) + .size_limit(1024 * 1024) + .dfa_size_limit(1024 * 1024) + .nest_limit(64) + .build() + .map_err(|_| "正则语法无效或复杂度超限，不支持前后顾与模式反向引用") + }) + .transpose()?; + if matches!( + rule.action, + PrivacyAction::RegexReplace | PrivacyAction::RenameKey + ) { + validate_replacement(&rule.replacement, regex.as_ref().ok_or("请填写匹配表达式")?)?; + } + if rule.action == PrivacyAction::SetValue && rule.pattern.is_some() { + return Err("固定值动作不使用匹配表达式"); + } + Ok(Self { + rule: rule.clone(), + path, + regex, + }) + } + + pub(super) fn apply( + &self, + value: &mut Value, + budget: &mut Budget<'_>, + ) -> Result { + let mut count = 0; + let removed = selector::visit(value, &self.path, budget, &mut |value, budget| { + self.apply_value(value, budget, &mut count) + })?; + if removed { + return Err("不能删除作用范围根节点"); + } + Ok(count) + } + + pub(super) fn apply_value( + &self, + value: &mut Value, + budget: &mut Budget<'_>, + count: &mut usize, + ) -> Result { + budget.step()?; + match self.rule.action { + PrivacyAction::RemoveField => { + if let Some(regex) = &self.regex { + let text = value.as_str().ok_or("条件删除只能匹配字符串值")?; + budget.text(text)?; + if !regex.is_match(text) { + return Ok(false); + } + } + *count += 1; + return Ok(true); + } + PrivacyAction::SetValue => { + if std::mem::discriminant(value) != std::mem::discriminant(&self.rule.value) { + return Err("固定值必须保持原字段类型"); + } + *value = self.rule.value.clone(); + *count += 1; + } + PrivacyAction::RegexReplace => { + let text = value.as_str().ok_or("正则替换只能处理字符串")?; + let (replacement, matches) = self.replace(text, budget)?; + *value = Value::String(replacement); + *count += matches; + } + PrivacyAction::RenameKey => { + let object = value.as_object().ok_or("字段名替换只能处理对象")?; + let mut keys = BTreeSet::new(); + let mut changes = Vec::new(); + for key in object.keys() { + budget.step()?; + let (replacement, matches) = self.replace(key, budget)?; + if !keys.insert(replacement.clone()) { + return Err("字段名替换产生重名"); + } + *count += matches; + changes.push(replacement); + } + let Value::Object(object) = std::mem::take(value) else { + return Err("目标类型已变化"); + }; + *value = Value::Object(changes.into_iter().zip(object.into_values()).collect()); + } + } + Ok(false) + } + + fn replace( + &self, + text: &str, + budget: &mut Budget<'_>, + ) -> Result<(String, usize), &'static str> { + budget.text(text)?; + let regex = self.regex.as_ref().ok_or("缺少已编译表达式")?; + let mut output = String::new(); + let mut end = 0; + let mut count = 0; + for captures in regex.captures_iter(text) { + budget.step()?; + let matched = captures.get(0).ok_or("缺少匹配结果")?; + output.push_str(&text[end..matched.start()]); + let largest = captures + .iter() + .flatten() + .map(|group| group.len()) + .max() + .unwrap_or(0); + let references = self + .rule + .replacement + .bytes() + .filter(|byte| *byte == b'$') + .count(); + if output + .len() + .saturating_add(self.rule.replacement.len()) + .saturating_add(largest.saturating_mul(references)) + > MAX_VALUE_BYTES + { + return Err("捕获组展开超出大小限制"); + } + captures.expand(&self.rule.replacement, &mut output); + if output.len() > MAX_VALUE_BYTES { + return Err("替换结果超出大小限制"); + } + end = matched.end(); + count += 1; + if !self.rule.replace_all { + break; + } + } + output.push_str(&text[end..]); + if output.len() > MAX_VALUE_BYTES { + return Err("替换结果超出大小限制"); + } + Ok((output, count)) + } +} + +fn validate_replacement(value: &str, regex: &Regex) -> Result<(), &'static str> { + let mut rest = value; + while let Some(index) = rest.find('$') { + rest = &rest[index + 1..]; + if let Some(tail) = rest.strip_prefix('$') { + rest = tail; + continue; + } + let (name, tail) = if let Some(tail) = rest.strip_prefix('{') { + tail.split_once('}').ok_or("捕获组引用缺少 }")? + } else { + let length = rest + .bytes() + .take_while(|c| c.is_ascii_alphanumeric() || *c == b'_') + .count(); + if length == 0 { + continue; + } + (&rest[..length], &rest[length..]) + }; + let valid = name.parse::().map_or_else( + |_| regex.capture_names().flatten().any(|group| group == name), + |index| index < regex.captures_len(), + ); + if !valid { + return Err("替换内容引用了不存在的捕获组"); + } + rest = tail; + } + Ok(()) +} diff --git a/backend/crates/providers/openai/src/transport/privacy/selector.rs b/backend/crates/providers/openai/src/transport/privacy/selector.rs new file mode 100644 index 000000000..8383beecb --- /dev/null +++ b/backend/crates/providers/openai/src/transport/privacy/selector.rs @@ -0,0 +1,113 @@ +//! 有界 JSON 字段选择器，仅支持成员、索引和单层通配 + +use serde_json::Value; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Segment { + Key(String), + Index(usize), + Wildcard, +} + +pub(super) fn parse(source: &str) -> Result, &'static str> { + if source.len() > 1024 || !source.starts_with('$') { + return Err("字段路径应以 $ 开头且不超过 1024 字节"); + } + let mut rest = &source[1..]; + let mut segments = Vec::new(); + while !rest.is_empty() { + if segments.len() == 16 { + return Err("字段路径最多 16 层"); + } + if let Some(tail) = rest.strip_prefix('.') { + let end = tail.find(['.', '[']).unwrap_or(tail.len()); + let key = &tail[..end]; + segments.push(if key == "*" { + Segment::Wildcard + } else if !key.is_empty() + && key + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-')) + { + Segment::Key(key.to_owned()) + } else { + return Err("成员名包含特殊字符时使用双引号键名"); + }); + rest = &tail[end..]; + } else if let Some(tail) = rest.strip_prefix('[') { + if tail.starts_with('"') { + let mut decoder = serde_json::Deserializer::from_str(tail).into_iter::(); + let key = decoder + .next() + .and_then(Result::ok) + .ok_or("无效的双引号键名")?; + rest = tail[decoder.byte_offset()..] + .strip_prefix(']') + .ok_or("字段路径缺少 ]")?; + segments.push(Segment::Key(key)); + } else { + let (value, tail) = tail.split_once(']').ok_or("字段路径缺少 ]")?; + segments.push(if value == "*" { + Segment::Wildcard + } else if !value.is_empty() && value.bytes().all(|c| c.is_ascii_digit()) { + Segment::Index(value.parse().map_err(|_| "数组索引超出范围")?) + } else { + return Err("仅支持数组索引和 * 通配，不支持过滤表达式"); + }); + rest = tail; + } + } else { + return Err("无效的字段路径"); + } + } + Ok(segments) +} + +/// 从后向前访问数组，删除匹配项不会改变尚未访问的索引 +pub(super) fn visit( + value: &mut Value, + segments: &[Segment], + budget: &mut super::Budget<'_>, + apply: &mut impl FnMut(&mut Value, &mut super::Budget<'_>) -> Result, +) -> Result { + budget.step()?; + let Some((first, rest)) = segments.split_first() else { + return apply(value, budget); + }; + match (first, value) { + (Segment::Key(key), Value::Object(object)) => { + if let Some(child) = object.get_mut(key) + && visit(child, rest, budget, apply)? + { + object.remove(key); + } + } + (Segment::Index(index), Value::Array(items)) => { + if let Some(child) = items.get_mut(*index) + && visit(child, rest, budget, apply)? + { + items.remove(*index); + } + } + (Segment::Wildcard, Value::Object(object)) => { + let keys: Vec<_> = object.keys().cloned().collect(); + for key in keys { + if let Some(child) = object.get_mut(&key) + && visit(child, rest, budget, apply)? + { + object.remove(&key); + } + } + } + (Segment::Wildcard, Value::Array(items)) => { + for index in (0..items.len()).rev() { + if visit(&mut items[index], rest, budget, apply)? { + items.remove(index); + } + } + } + // 缺少字段属于正常未命中，已存在却不符合路径类型属于错误 + _ => return Err("目标结构不符合字段路径"), + } + Ok(false) +} diff --git a/backend/crates/providers/openai/src/transport/request.rs b/backend/crates/providers/openai/src/transport/request.rs index 5f0ed91e7..295c33c54 100644 --- a/backend/crates/providers/openai/src/transport/request.rs +++ b/backend/crates/providers/openai/src/transport/request.rs @@ -612,6 +612,17 @@ fn metadata_string(request: &CodexResponsesRequest, key: &str) -> Option .map(ToOwned::to_owned) } +pub(crate) fn serialize_ascii_turn_metadata(value: &Value) -> Option { + let mut bytes = Vec::new(); + value + .serialize(&mut serde_json::Serializer::with_formatter( + &mut bytes, + AsciiTurnMetadataFormatter, + )) + .ok()?; + String::from_utf8(bytes).ok() +} + pub(crate) fn scope_turn_metadata( raw: &str, installation_id: &str, @@ -648,14 +659,7 @@ pub(crate) fn scope_turn_metadata( // Codex 的 turn metadata 同时承载于 HTTP header 与 WS client_metadata // 改写安装 ID 后仍须保持官方 to_ascii_json_string 的编码合同；普通 // to_string 会把中文工作区路径还原成 UTF-8，触发上游 WS metadata 后 Close 1000 - let mut bytes = Vec::new(); - metadata - .serialize(&mut serde_json::Serializer::with_formatter( - &mut bytes, - AsciiTurnMetadataFormatter, - )) - .ok()?; - String::from_utf8(bytes).ok() + serialize_ascii_turn_metadata(&Value::Object(metadata)) } struct AsciiTurnMetadataFormatter; diff --git a/backend/crates/providers/openai/tests/provider/contract/mod.rs b/backend/crates/providers/openai/tests/provider/contract/mod.rs index 5c18742be..dc0152e16 100644 --- a/backend/crates/providers/openai/tests/provider/contract/mod.rs +++ b/backend/crates/providers/openai/tests/provider/contract/mod.rs @@ -7,6 +7,7 @@ mod image_account_eligibility; mod image_model_access; mod passthrough; mod precommit; +mod privacy; mod response_interrupt; mod session_binding; mod timing; diff --git a/backend/crates/providers/openai/tests/provider/contract/privacy.rs b/backend/crates/providers/openai/tests/provider/contract/privacy.rs new file mode 100644 index 000000000..c5147c3cd --- /dev/null +++ b/backend/crates/providers/openai/tests/provider/contract/privacy.rs @@ -0,0 +1,331 @@ +//! 隐私规则在真实 HTTP、WebSocket 和发送前拒绝边界的合同 + +use super::*; +use crate::transport::privacy::{policy, rule}; +use gateway_core::settings::privacy::{CodexPrivacyPolicy, PrivacyAction, PrivacyScope}; + +fn privacy_context(policy: &CodexPrivacyPolicy, fallback: bool) -> AttemptContext { + let context = AttemptContext::new( + RequestAttemptContext::new( + ModelRequestId::new("req_privacy").unwrap(), + ClientApiKeyId::new("key_openai_contract").unwrap(), + ) + .with_privacy(Some( + provider_openai::transport::privacy::compile(policy).unwrap(), + )), + NonZeroU32::MIN, + SystemTime::now() + Duration::from_secs(30), + account_policy(), + AccountAttemptContext::new(BTreeSet::new(), None, None) + .with_account_scope(contract_account_scope()), + None, + CancellationToken::new(), + ); + if fallback { + context.with_transport(AttemptTransport::Fallback) + } else { + context + } +} + +fn outbound_policy() -> CodexPrivacyPolicy { + let mut auth = rule( + PrivacyScope::RequestHeader, + "authorization", + PrivacyAction::SetValue, + ); + auth.value = json!("Bearer configured-synthetic-token"); + let mut input = rule( + PrivacyScope::RequestBody, + "$.input[0].content[0].text", + PrivacyAction::RegexReplace, + ); + input.id = "input".into(); + input.pattern = Some("hello".into()); + input.replacement = "hellohello".into(); + let mut metadata = rule( + PrivacyScope::TurnMetadata, + "$.workspaces", + PrivacyAction::RemoveField, + ); + metadata.id = "metadata".into(); + policy(vec![auth, input, metadata]) +} + +fn metadata_operation() -> Operation { + let header = + json!({"workspaces":{"/home/alex/private":{}},"request_kind":"turn","header_only":true}); + let client = json!({"workspaces":{"/home/alex/private":{}},"tool_namespaces_info":{"tools":["example"]}}); + let body = json!({"model":"gpt-5.4","input":"hello","turnMetadata":header.to_string(),"client_metadata":{"x-codex-turn-metadata":client.to_string()}}); + Operation::Generate(GenerateRequest::from_protocol_payload( + ProtocolPayload::json_object("openai", body.as_object().unwrap().clone()).unwrap(), + )) +} + +fn assert_metadata_filtered(raw: &str, client_copy: bool) { + let metadata: Value = serde_json::from_str(raw).unwrap(); + assert!(metadata.get("workspaces").is_none()); + if client_copy { + assert_eq!( + metadata["tool_namespaces_info"], + json!({"tools":["example"]}) + ); + assert!(metadata.get("header_only").is_none()); + } else { + assert_eq!(metadata["header_only"], true); + assert!(metadata.get("tool_namespaces_info").is_none()); + } +} + +#[tokio::test] +async fn privacy_json_endpoints_preserve_untouched_bytes_and_rewrite_matched_body() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"output":"ok"}))) + .mount(&server) + .await; + let provider = provider_with_base_url(&store, server.uri()); + let raw = Bytes::from_static( + br#"{ "id":"search", "workspaces":{"/private":{}}, "future":1, "future":2 }"#, + ); + for change_body in [false, true] { + let mut config = outbound_policy(); + config.rules.truncate(1); + if change_body { + let mut remove = rule( + PrivacyScope::RequestBody, + "$.workspaces", + PrivacyAction::RemoveField, + ); + remove.id = "remove-workspace".into(); + config.rules.push(remove); + } + let operation = Operation::Search(StandaloneSearchRequest::from_raw_json( + RawJsonPayload::new("openai", raw.clone()).unwrap(), + )); + let mut stream = provider + .clone() + .execute( + planned_provider_endpoint_request("openai", operation), + privacy_context(&config, false), + ) + .await + .unwrap(); + while let Some(event) = stream.next().await { + event.unwrap(); + } + let requests = server.received_requests().await.unwrap(); + let sent = requests.last().unwrap(); + assert_eq!( + sent.headers["authorization"], + "Bearer configured-synthetic-token" + ); + if change_body { + assert_eq!( + serde_json::from_slice::(&sent.body).unwrap(), + json!({"id":"search","future":2}) + ); + } else { + assert_eq!(sent.body, raw); + } + } +} + +#[tokio::test] +async fn privacy_rewrites_final_http_headers_and_body_once_per_attempt() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(CAPTURE_COMPLETED_SSE), + ) + .mount(&server) + .await; + let provider = provider_with_base_url(&store, server.uri()); + let operation = metadata_operation(); + for _ in 0..2 { + let mut stream = provider + .clone() + .execute( + planned_request("openai", operation.clone()), + privacy_context(&outbound_policy(), true), + ) + .await + .unwrap(); + while let Some(event) = stream.next().await { + event.unwrap(); + } + } + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + for request in requests { + assert_metadata_filtered( + request.headers["x-codex-turn-metadata"].to_str().unwrap(), + false, + ); + assert_metadata_filtered( + captured_request_body(&request)["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .unwrap(), + true, + ); + assert_eq!( + request.headers["authorization"], + "Bearer configured-synthetic-token" + ); + assert_eq!( + captured_request_body(&request)["input"][0]["content"][0]["text"], + "hellohello" + ); + } +} + +#[tokio::test] +async fn privacy_rejection_is_not_sent_and_cannot_retry_or_rotate() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let server = MockServer::start().await; + let mut invalid_target = rule( + PrivacyScope::RequestBody, + "$.input[0].content[0].text", + PrivacyAction::SetValue, + ); + invalid_target.value = json!(42); + let provider = provider_with_base_url(&store, server.uri()); + let result = provider + .execute( + planned_request("openai", generate_operation()), + privacy_context(&policy(vec![invalid_target]), true), + ) + .await; + let error = match result { + Err(error) => error, + Ok(mut stream) => stream.next().await.unwrap().err().unwrap(), + }; + assert_eq!(error.kind(), ProviderErrorKind::InvalidRequest); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + assert!(error.retry_is_prohibited()); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[tokio::test] +async fn privacy_rewrites_websocket_handshake_and_frame_once() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let captured = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = + crate::transport::accept_codex_test_websocket_with(stream, |request, response| { + assert_eq!( + request.headers()["authorization"], + "Bearer configured-synthetic-token" + ); + assert_metadata_filtered( + request.headers()["x-codex-turn-metadata"].to_str().unwrap(), + false, + ); + response.headers_mut().insert( + "sec-websocket-extensions", + "permessage-deflate".parse().unwrap(), + ); + }) + .await; + let frame = socket.next().await.unwrap().unwrap().into_text().unwrap(); + let body: Value = serde_json::from_str(&frame).unwrap(); + assert_metadata_filtered( + body["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .unwrap(), + true, + ); + assert_eq!(body["input"][0]["content"][0]["text"], "hellohello"); + socket.send(Message::Text(json!({"type":"response.completed","response":{"id":"resp_privacy","model":"gpt-5.4","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}).to_string().into())).await.unwrap(); + }); + let provider = provider_with_base_url(&store, format!("http://{address}")); + let mut stream = provider + .execute( + planned_request("openai", metadata_operation()), + privacy_context(&outbound_policy(), false), + ) + .await + .unwrap(); + while let Some(event) = stream.next().await { + event.unwrap(); + } + captured.await.unwrap(); +} + +#[tokio::test] +async fn privacy_header_change_uses_a_new_websocket_pool_connection() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let captured = tokio::spawn(async move { + let mut sockets = Vec::new(); + for expected in [ + "Bearer first-synthetic-token", + "Bearer second-synthetic-token", + ] { + let (stream, _) = timeout(Duration::from_secs(5), listener.accept()) + .await + .unwrap() + .unwrap(); + let mut socket = + crate::transport::accept_codex_test_websocket_with(stream, |request, response| { + assert_eq!(request.headers()["authorization"], expected); + response.headers_mut().insert( + "sec-websocket-extensions", + "permessage-deflate".parse().unwrap(), + ); + }) + .await; + let frame = socket.next().await.unwrap().unwrap().into_text().unwrap(); + assert_eq!( + serde_json::from_str::(&frame).unwrap()["input"][0]["content"][0]["text"], + "hellohello" + ); + socket.send(Message::Text(json!({"type":"response.completed","response":{"id":"resp_privacy_pool","model":"gpt-5.4","status":"completed","output":[]}}).to_string().into())).await.unwrap(); + // 保留第一条连接，确保下一次握手是池隔离的结果而非断线重连 + sockets.push(socket); + } + sockets + }); + let provider = provider_with_base_url(&store, format!("http://{address}")); + for token in [ + "Bearer first-synthetic-token", + "Bearer second-synthetic-token", + ] { + let mut config = outbound_policy(); + config.rules[0].value = json!(token); + let operation = Operation::Generate(generate_with_session_context( + "privacy-pool", + Some("privacy-thread"), + None, + )); + let mut stream = provider + .clone() + .execute( + planned_request("openai", operation), + privacy_context(&config, false), + ) + .await + .unwrap(); + timeout(Duration::from_secs(5), async { + while let Some(event) = stream.next().await { + event.unwrap(); + } + }) + .await + .unwrap(); + } + captured.await.unwrap(); +} diff --git a/backend/crates/providers/openai/tests/transport/mod.rs b/backend/crates/providers/openai/tests/transport/mod.rs index d60d011ea..b28de67cc 100644 --- a/backend/crates/providers/openai/tests/transport/mod.rs +++ b/backend/crates/providers/openai/tests/transport/mod.rs @@ -58,6 +58,7 @@ mod endpoints; mod headers; mod http_client; mod latency; +pub(crate) mod privacy; mod profile; mod profile_avatar; mod profile_contract; diff --git a/backend/crates/providers/openai/tests/transport/privacy/mod.rs b/backend/crates/providers/openai/tests/transport/privacy/mod.rs new file mode 100644 index 000000000..4c41fc8f1 --- /dev/null +++ b/backend/crates/providers/openai/tests/transport/privacy/mod.rs @@ -0,0 +1,385 @@ +//! 隐私规则的多承载一致性、自由字段配置、条件删除与有界失败合同 + +use gateway_core::settings::privacy::*; +use provider_openai::transport::privacy::{compile, preview}; +use serde_json::{Value, json}; + +pub(crate) fn rule(scope: PrivacyScope, selector: &str, action: PrivacyAction) -> PrivacyRule { + PrivacyRule { + id: "rule-1".into(), + name: "测试规则".into(), + enabled: true, + scope, + selector: selector.into(), + action, + pattern: None, + replacement: String::new(), + value: Value::Null, + replace_all: true, + case_insensitive: false, + multi_line: false, + } +} + +pub(crate) fn policy(rules: Vec) -> CodexPrivacyPolicy { + CodexPrivacyPolicy { + enabled: true, + on_error: PrivacyFailureMode::RejectRequest, + rules, + } +} + +fn sample(rules: Vec, body: Value) -> PrivacyPreviewRequest { + PrivacyPreviewRequest { + policy: policy(rules), + body, + headers: Default::default(), + turn_metadata: None, + } +} + +#[test] +fn privacy_removes_workspaces_from_every_copy_without_merging_metadata() { + let workspaces = json!({"/home/alex/项目": {"associated_remote_urls": {"origin":"https://example.test/private/repo"}}}); + let header = json!({"workspaces":workspaces,"request_kind":"primary"}); + let client = json!({"workspaces":workspaces,"tool_namespaces_info":{"tools":["a"]}}); + let mut request = sample( + vec![rule( + PrivacyScope::TurnMetadata, + "$.workspaces", + PrivacyAction::RemoveField, + )], + json!({ + "turnMetadata": header.to_string(), "turn_metadata": header, "x-codex-turn-metadata": header, + "client_metadata": {"x-codex-turn-metadata":client.to_string()} + }), + ); + request.turn_metadata = Some(header.to_string()); + request.headers.insert( + "x-codex-turn-metadata".into(), + vec![header.to_string(), header.to_string()], + ); + let output = preview(request).unwrap(); + assert_eq!(output.outcomes[0].matches, 7); + assert_eq!( + output.turn_metadata.as_deref(), + Some("{\"request_kind\":\"primary\"}") + ); + assert_eq!( + output.body["turn_metadata"], + json!({"request_kind":"primary"}) + ); + assert_eq!( + serde_json::from_str::( + output.body["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .unwrap() + ) + .unwrap(), + json!({"tool_namespaces_info":{"tools":["a"]}}) + ); + assert!( + output.headers["x-codex-turn-metadata"] + .iter() + .all(|raw| !raw.contains("workspaces")) + ); +} + +#[test] +fn privacy_conditionally_deletes_values_and_renames_path_keys_in_sequence() { + let mut remove = rule( + PrivacyScope::RequestBody, + "$.workspaces.*.associated_remote_urls.*", + PrivacyAction::RemoveField, + ); + remove.pattern = Some("/private-org/".into()); + let mut rename = rule( + PrivacyScope::RequestBody, + "$.workspaces", + PrivacyAction::RenameKey, + ); + rename.id = "rename".into(); + rename.pattern = Some("^/home/[^/]+/(.*)$".into()); + rename.replacement = "/workspace/${1}".into(); + let output = preview(sample(vec![remove, rename], json!({"workspaces":{ + "/home/alex/demo":{"associated_remote_urls":{"origin":"https://host/private-org/demo","upstream":"https://host/public/demo"}} + }}))).unwrap(); + assert_eq!( + output.body, + json!({"workspaces":{"/workspace/demo":{"associated_remote_urls":{"upstream":"https://host/public/demo"}}}}) + ); + let mut remove = rule( + PrivacyScope::RequestBody, + "$.remotes[*]", + PrivacyAction::RemoveField, + ); + remove.pattern = Some("private".into()); + assert_eq!( + preview(sample( + vec![remove], + json!({"remotes":["private-a","public","private-b","private-c"]}) + )) + .unwrap() + .body, + json!({"remotes":["public"]}) + ); +} + +#[test] +fn privacy_has_no_field_protection_list_in_any_scope() { + for name in [ + "authorization", + "cookie", + "chatgpt-account-id", + "x-codex-turn-state", + "x-codex-installation-id", + ] { + let mut request = sample( + vec![rule( + PrivacyScope::RequestHeader, + name, + PrivacyAction::RemoveField, + )], + json!({}), + ); + request + .headers + .insert(name.into(), vec!["synthetic-value".into()]); + assert!(preview(request).unwrap().headers.is_empty(), "{name}"); + } + for name in [ + "session_id", + "thread_id", + "previous_response_id", + "prompt_cache_key", + "tool_namespaces_info", + "history_ingest_requested", + "analytics_enabled", + ] { + let output = preview(sample( + vec![rule( + PrivacyScope::RequestBody, + &format!("$.{name}"), + PrivacyAction::RemoveField, + )], + json!({name:"synthetic"}), + )) + .unwrap(); + assert_eq!(output.body, json!({}), "{name}"); + } +} + +#[test] +fn privacy_rolls_back_all_copies_of_a_failed_rule_and_continues_in_order() { + let mut first = rule( + PrivacyScope::TurnMetadata, + "$.workspace", + PrivacyAction::RegexReplace, + ); + first.pattern = Some("private".into()); + first.replacement = "public".into(); + let mut second = rule(PrivacyScope::RequestBody, "$.keep", PrivacyAction::SetValue); + second.id = "second".into(); + second.value = json!("done"); + let body = json!({"turnMetadata":{"workspace":"private"},"client_metadata":{"turnMetadata":{"workspace":42}},"keep":"start"}); + let mut request = sample(vec![first.clone(), second], body.clone()); + request.policy.on_error = PrivacyFailureMode::SkipRule; + let output = preview(request).unwrap(); + assert_eq!(output.body["turnMetadata"], body["turnMetadata"]); + assert_eq!(output.body["keep"], "done"); + assert_eq!(output.outcomes[0].status, "skipped"); + assert_eq!(output.outcomes[1].status, "applied"); + assert_eq!( + preview(sample(vec![first], body)).err().unwrap().rule_index, + 0 + ); +} + +#[test] +fn privacy_key_collisions_fail_without_losing_values() { + let mut rename = rule( + PrivacyScope::RequestBody, + "$.workspaces", + PrivacyAction::RenameKey, + ); + rename.pattern = Some(".+".into()); + rename.replacement = "same".into(); + let body = json!({"workspaces":{"first":1,"second":2}}); + let mut request = sample(vec![rename], body.clone()); + request.policy.on_error = PrivacyFailureMode::SkipRule; + let output = preview(request).unwrap(); + assert_eq!(output.body, body); + assert_eq!(output.outcomes[0].status, "skipped"); +} + +#[test] +fn privacy_desktop_context_is_classified_and_reencoded_without_touching_prose() { + let text = "{\"workspaces\":{\"/private\":{}},\"note\":\"a & b\"}"; + let mut replacement = rule( + PrivacyScope::DesktopGitContext, + "$.workspaces", + PrivacyAction::RemoveField, + ); + let body = json!({"input":[{"role":"user","content":[{"type":"input_text","text":text},{"type":"input_text","text":text}],"internal_chat_message_metadata_passthrough":{"content_item_kinds":["additional_content.codex_git_state","user.text"]}}]}); + let output = preview(sample(vec![replacement.clone()], body)).unwrap(); + assert!( + !output.body["input"][0]["content"][0]["text"] + .as_str() + .unwrap() + .contains("workspaces") + ); + assert!( + output.body["input"][0]["content"][0]["text"] + .as_str() + .unwrap() + .contains("a & b") + ); + assert_eq!(output.body["input"][0]["content"][1]["text"], text); + replacement.scope = PrivacyScope::EnvironmentText; + replacement.selector = "$".into(); + replacement.action = PrivacyAction::RegexReplace; + replacement.pattern = Some("Asia/Shanghai".into()); + replacement.replacement = "UTC".into(); + let body = json!({"input":[{"role":"developer","content":[{"type":"input_text","text":"Asia/Shanghai"}]}]}); + assert!( + preview(sample(vec![replacement], body)) + .unwrap() + .body + .to_string() + .contains("UTC") + ); +} + +#[test] +fn privacy_validates_regex_paths_captures_and_execution_limits() { + let mut replace = rule( + PrivacyScope::RequestBody, + "$[\"a.b\"][0]", + PrivacyAction::RegexReplace, + ); + replace.pattern = Some("(?Pa)".into()); + replace.replacement = "${word}${1}$$".into(); + assert_eq!( + preview(sample(vec![replace.clone()], json!({"a.b":["a"]}))) + .unwrap() + .body, + json!({"a.b":["aa$"]}) + ); + replace.replacement = "${missing}".into(); + assert!(compile(&policy(vec![replace.clone()])).is_err()); + replace.replacement.clear(); + replace.pattern = Some("(?<=a)b".into()); + assert!(compile(&policy(vec![replace])).is_err()); + for path in ["$..secret", "$[?(@.secret)]", "$['single']"] { + assert!( + compile(&policy(vec![rule( + PrivacyScope::RequestBody, + path, + PrivacyAction::RemoveField + )])) + .is_err() + ); + } + let policy = compile(&policy(vec![rule( + PrivacyScope::RequestBody, + "$.items.*.missing", + PrivacyAction::RemoveField, + )])) + .unwrap(); + let mut body = json!({"items":vec![json!({}); 10_001]}); + let original = body.clone(); + let error = policy + .apply(&mut body, &mut Default::default(), &mut None, &|| false) + .unwrap_err(); + assert_eq!(error.reason, "规则执行预算超限"); + assert_eq!(body, original); + assert!( + policy + .apply(&mut json!({}), &mut Default::default(), &mut None, &|| true) + .is_err() + ); +} + +#[test] +fn privacy_disabled_policy_and_absent_fields_preserve_original_bytes() { + let body = json!({"turnMetadata":"{ \"keep\" : 1 }"}); + let request = sample( + vec![rule( + PrivacyScope::TurnMetadata, + "$.workspaces", + PrivacyAction::RemoveField, + )], + body.clone(), + ); + assert_eq!(preview(request.clone()).unwrap().body, body); + let mut request = request; + request.policy.enabled = false; + assert_eq!(preview(request).unwrap().outcomes[0].status, "disabled"); +} + +#[test] +fn privacy_context_parser_handles_quoted_brackets_and_bounds_xml_depth() { + let text = "b\">"; + let body = json!({"input":[{"role":"user","content":[{"type":"input_text","text":text}]}]}); + let output = preview(sample( + vec![rule( + PrivacyScope::DesktopGitContext, + "$.workspaces", + PrivacyAction::RemoveField, + )], + body, + )) + .unwrap(); + let changed = output.body["input"][0]["content"][0]["text"] + .as_str() + .unwrap(); + let document = roxmltree::Document::parse(changed).unwrap(); + let git = document + .descendants() + .find(|node| node.has_tag_name("git")) + .unwrap(); + assert_eq!(git.attribute("source"), Some("a>b")); + assert_eq!( + serde_json::from_str::(git.text().unwrap()).unwrap(), + json!({"note":"a & b"}) + ); + let nested = format!( + "{}value{}", + "".repeat(100), + "".repeat(100) + ); + let mut replace = rule( + PrivacyScope::EnvironmentText, + "$", + PrivacyAction::RegexReplace, + ); + replace.pattern = Some("value".into()); + replace.replacement = "changed".into(); + let body = json!({"input":[{"role":"user","content":[{"type":"input_text","text":nested}]}]}); + assert_eq!( + preview(sample(vec![replace], body)).err().unwrap().reason, + "上下文 XML 嵌套超过 64 层" + ); +} + +#[test] +fn privacy_reports_headers_that_websocket_opening_cannot_serialize() { + let mut replace = rule( + PrivacyScope::RequestHeader, + "x-label", + PrivacyAction::SetValue, + ); + replace.value = json!("别名"); + let mut request = sample(vec![replace], json!({})); + request + .headers + .insert("x-label".into(), vec!["original".into()]); + assert_eq!( + preview(request.clone()).err().unwrap().reason, + "替换结果必须是 ASCII 请求头值" + ); + request.policy.on_error = PrivacyFailureMode::SkipRule; + let result = preview(request).unwrap(); + assert_eq!(result.headers["x-label"], vec!["original"]); + assert_eq!(result.outcomes[0].status, "skipped"); +} diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 925142d9f..2708c977b 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -23,3 +23,4 @@ e8c1339166ac02fb2d4a077424ee5f64f6dc20eb4d3f8ec4fcb09c0f2a0e61f6 0020_plugin_tr d431ebf24669451da06bb6671136bf58cddcc713b43a1a705eb33f9cb4f9b577 0023_error_details_and_account_affinity.sql fa99dde0fe0b3d7da6693d76b2735629dde0dc255ea8c53f0354857a4e80669b 0024_preferred_account_affinity.sql 299d347add09b1e6f086732c0db8269efb640db4ecccb948e17c067b572afc43 0025_request_observation.sql +d4b599066d7382a1c9cb6b3c003200046579e4fce3531f232cbd665a9d92c542 0026_codex_privacy_policy.sql diff --git a/backend/migrations/0026_codex_privacy_policy.sql b/backend/migrations/0026_codex_privacy_policy.sql new file mode 100644 index 000000000..0be3a97fc --- /dev/null +++ b/backend/migrations/0026_codex_privacy_policy.sql @@ -0,0 +1,11 @@ +-- 隐私规则默认关闭，保留既有请求行为 +alter table runtime_settings + add column codex_privacy_policy_json jsonb not null + default '{"enabled":false,"onError":"skip_rule","rules":[]}'::jsonb + check ( + jsonb_typeof(codex_privacy_policy_json) = 'object' + and jsonb_typeof(codex_privacy_policy_json->'enabled') = 'boolean' + and codex_privacy_policy_json->>'onError' in ('skip_rule', 'reject_request') + and jsonb_typeof(codex_privacy_policy_json->'rules') = 'array' + and codex_privacy_policy_json ?& array['enabled', 'onError', 'rules'] + ); diff --git a/docs/api.md b/docs/api.md index 8eecbf6c9..420dbccd4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1322,6 +1322,7 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly | --- | --- | --- | | `GET` | `/api/admin/settings` | 读取运行设置 | | `POST` | `/api/admin/settings/update` | 原子保存运行设置；身份配置按显式提供的 Provider 项更新 | +| `POST` | `/api/admin/settings/privacy/preview` | 使用管理员提供的样本预览隐私规则，不保存设置或样本 | | `GET` | `/api/admin/settings/client-downloads/codex-desktop/windows` | 提取 Codex Desktop Windows 离线安装直链；`refresh=true` 强制刷新进程内短缓存 | | `GET` | `/api/admin/settings/client-profiles/{provider}` | 读取 `openai` 或 `xai` 的可选配置和 `globalConfiguration` | | `POST` | `/api/admin/settings/client-profiles/{provider}/preview` | `{ configuration }`，完整配置对象或 `null`（解析当前通用设置）；只预览，不保存 | @@ -1340,6 +1341,7 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly | 并发控制版本 | `configRevision` | | [客户端身份](#provider-客户端身份) | `providerRequestProfiles`、`openaiClientProfile`、`xaiClientProfile` | | [请求位置](#请求位置) | `requestLocationEnabled`、`requestLocation` | +| [隐私策略](#隐私策略) | `codexPrivacyPolicy` | | 模型映射 | `modelMappings` | | 凭据刷新 | `refreshMarginSeconds`、`refreshConcurrency` | | [并发与排队](#并发与排队) | `maxConcurrentPerAccount`、`maxWaitingPerKey`、`maxWaitingPerAccount`、`openaiGuardianReservedConcurrency`、`concurrencyWaitTimeoutSeconds`、`requestIntervalMs` | @@ -1369,6 +1371,47 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly 关联代理配置了自定义位置时优先使用代理值。保存后通过现有配置发布机制对新请求生效， 已开始请求及其重试保持同一份全局值；普通文本、绝对时间戳和数据驻留要求不受影响 +### 隐私策略 + +`codexPrivacyPolicy` 为必填对象，默认 `{ enabled: false, onError: "skip_rule", rules: [] }`。 +只处理经网关发送的 OpenAI Responses HTTP／WebSocket 和独立 JSON 端点请求，不控制客户端直接发送的遥测。 +规则按列表顺序执行，关闭策略或单条规则会保留配置。没有字段保护名单，认证、会话、工具等字段也可配置；改写后对上游行为的影响由配置者承担 + +每条规则包含唯一 `id`、`name`、`enabled`、`scope`、`selector`、`action`、`pattern`、`replacement`、 +`value`、`replaceAll`、`caseInsensitive`、`multiLine`。作用范围如下： + +| `scope` | 选择对象 | +| --- | --- | +| `turn_metadata` | 请求头、正文及 `client_metadata` 中的 turn metadata 对象，各副本独立处理 | +| `desktop_git_context` | 自动附加的 Desktop Git 上下文中解码后的 JSON 对象 | +| `environment_text` | 已识别的环境上下文或 Desktop 时间上下文文本，`selector` 为 `$` | +| `request_body` | 最终出站 JSON 正文 | +| `request_header` | 最终业务请求头，`selector` 直接填写头名称，重复头逐个处理 | + +JSON 路径支持 `$`、`.成员`、`["带特殊字符的键"]`、`[索引]`、`.*` 和 `[*]`，不支持递归和过滤表达式。 +缺少字段视为未命中；类型不符属于执行失败。`regex_replace` 替换字符串，`set_value` 保持原字段类型设置 JSON 值， +`rename_key` 替换所选对象的直接键名，重名时失败；`remove_field` 删除所选字段或数组项，不能删除作用范围根节点。 +删除动作的 `pattern: null` 表示无条件删除，字符串模式表示只删除字符串值匹配的字段或数组项。 +请求头规则生成的值必须是 ASCII 文本，以保持 HTTP 与 WebSocket opening 的一致性；metadata 范围会自动转义 Unicode + +例如，`turn_metadata` 范围的 `$.workspaces` 配合 `remove_field` 删除整个工作区映射； +`$.workspaces.*.associated_remote_urls.*` 配合 `remove_field` 和 `pattern: "private-org"` 只删除命中的远端； +`$.workspaces` 配合 `rename_key` 可改写工作区路径键名。环境文本替换必须保留 XML 包裹结构 + +正则使用 Rust `regex` 语法，不支持前后顾或模式反向引用；替换内容支持 `${1}`、`${name}` 和字面量 `$$`。 +`replaceAll` 控制首个或全部匹配，另支持忽略大小写和多行锚点。最多 32 条规则，路径最多 16 层、1024 字节， +模式与替换内容分别最多 4096 字节，固定值编码后最多 4096 字节。XML 上下文最多 10000 个节点，环境文本最多 64 层。 +执行限制为正文 32 MiB、单目标文本及编码后的上下文 1 MiB、 +累计处理文本 16 MiB、10000 次访问与匹配步骤、100 ms 检查预算；时间预算在处理检查点检查，不是硬实时中断 + +执行失败时，`skip_rule` 撤销本条规则在所有副本上的修改并继续，意味着本条脱敏未完成； +`reject_request` 阻止发送并返回本地请求错误，不因该错误自动重试或换号。设置保存时校验所有规则，包含未启用的规则 + +预览请求为 `{ policy, body, headers, turnMetadata }`，其中 `headers` 为头名称到字符串数组的映射， +`turnMetadata` 为编码后的 JSON 字符串或 `null`。正文最多 256 KiB，请求头与独立 metadata 合计最多 256 KiB。 +响应返回处理后的三个承载值，以及 `outcomes: [{ ruleId, matches, status, reason }]`； +状态为 `disabled`、`unmatched`、`applied` 或 `skipped`，错误不回显请求内容或表达式 + ### 并发与排队 `maxConcurrentPerAccount` 是默认账号并发上限，取值 0～4294967295；`0` 表示不限制。 diff --git a/docs/architecture.md b/docs/architecture.md index b1c1b6372..48247ed32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -293,6 +293,12 @@ service；客户端原生对象保存在有界进程缓存中，与套餐 Redis `RuntimeSnapshot → RoutingPlan → AttemptContext` 冻结传递，关闭时保留客户端原有字段； Provider 在选定账号后应用代理位置覆盖。请求期间不额外查询全局设置，配置发布不改变已开始请求的全局值 +隐私规则合同及编译端口由 Core 定义，OpenAI Provider 拥有路径、正则和承载结构的解释，Admin 的保存校验与样本预览复用同一实现。 +Store 将策略与设置 revision 原子持久化；编译结果随快照、计划及 attempt 冻结，未改变策略的请求设置覆盖复用已编译对象。 +规则在宿主请求头和 WS metadata 投影完成后作用于本次出站副本，重试及传输回退从原请求重新构造，避免重复替换。 +每条规则跨承载副本原子提交；发送前拒绝携带 `NotSent` 和禁止重试标记。被规则改写的握手头参与 WS 池身份，防止复用旧值， +关闭策略不进入改写管线。字段与执行限制见[隐私策略 API](api.md#隐私策略) + Fast 限制由同一快照链路冻结：Client Key 所有绑定分组的开关取逻辑或； 禁用分组仍贡献限制，无分组 Key 不限制 Fast，不按所选账号的分组重新解释。 OpenAI Provider 在独立编码请求上、生成上游头与观测前统一应用顶层档位覆盖，HTTP、WS 与重试共用。 diff --git a/frontend/src/api/modules/settings/index.ts b/frontend/src/api/modules/settings/index.ts index cb7a09070..3c05c87c9 100644 --- a/frontend/src/api/modules/settings/index.ts +++ b/frontend/src/api/modules/settings/index.ts @@ -1,9 +1,11 @@ import type { RequestOptions } from '../../request' +import type { CodexPrivacyPolicy } from './privacy' import type { ClientProfileSelection, ProviderRequestProfiles, ProviderRequestProfileUpdates, XaiClientProfileSelection } from './profiles' import request from '../../request' export * from './backups' export * from './pricing' +export * from './privacy' export * from './system' export interface RequestLocation { @@ -35,6 +37,7 @@ export interface RuntimeSettings { openaiClientProfile: ClientProfileSelection | null xaiClientProfile: XaiClientProfileSelection | null + codexPrivacyPolicy: CodexPrivacyPolicy requestLocationEnabled: boolean requestLocation: RequestLocation modelMappings: Record diff --git a/frontend/src/api/modules/settings/privacy.ts b/frontend/src/api/modules/settings/privacy.ts new file mode 100644 index 000000000..17b88b05b --- /dev/null +++ b/frontend/src/api/modules/settings/privacy.ts @@ -0,0 +1,46 @@ +import type { RequestOptions } from '../../request' +import request from '../../request' + +export type PrivacyScope = 'turn_metadata' | 'desktop_git_context' | 'environment_text' | 'request_body' | 'request_header' +export type PrivacyAction = 'regex_replace' | 'set_value' | 'rename_key' | 'remove_field' +export type JsonValue = unknown + +export interface PrivacyRule { + id: string + name: string + enabled: boolean + scope: PrivacyScope + selector: string + action: PrivacyAction + pattern: string | null + replacement: string + value: JsonValue + replaceAll: boolean + caseInsensitive: boolean + multiLine: boolean +} + +export interface CodexPrivacyPolicy { + enabled: boolean + onError: 'skip_rule' | 'reject_request' + rules: PrivacyRule[] +} + +export interface PrivacySample { + body: JsonValue + headers: Record + turnMetadata: string | null +} + +export interface PrivacyPreviewResult extends PrivacySample { + outcomes: { ruleId: string, matches: number, status: string, reason: string | null }[] +} + +export function previewPrivacyPolicy(data: PrivacySample & { policy: CodexPrivacyPolicy }, options: RequestOptions = {}) { + return request({ + url: '/api/admin/settings/privacy/preview', + method: 'POST', + data, + ...options, + }) +} diff --git a/frontend/src/views/settings/components/AccountAffinityCard.vue b/frontend/src/views/settings/components/AccountAffinityCard.vue index 19dd2b13d..e69c38a56 100644 --- a/frontend/src/views/settings/components/AccountAffinityCard.vue +++ b/frontend/src/views/settings/components/AccountAffinityCard.vue @@ -1,6 +1,6 @@ + + diff --git a/frontend/src/views/settings/components/privacy/PrivacyPolicyCard.vue b/frontend/src/views/settings/components/privacy/PrivacyPolicyCard.vue new file mode 100644 index 000000000..8ad24f8f4 --- /dev/null +++ b/frontend/src/views/settings/components/privacy/PrivacyPolicyCard.vue @@ -0,0 +1,116 @@ + + + diff --git a/frontend/src/views/settings/components/privacy/RuleEditor.vue b/frontend/src/views/settings/components/privacy/RuleEditor.vue new file mode 100644 index 000000000..dfad5bb2a --- /dev/null +++ b/frontend/src/views/settings/components/privacy/RuleEditor.vue @@ -0,0 +1,208 @@ + + + diff --git a/frontend/src/views/settings/components/privacy/RulesTable.vue b/frontend/src/views/settings/components/privacy/RulesTable.vue new file mode 100644 index 000000000..4e7cbbf57 --- /dev/null +++ b/frontend/src/views/settings/components/privacy/RulesTable.vue @@ -0,0 +1,82 @@ + + + diff --git a/frontend/src/views/settings/components/privacy/presets.ts b/frontend/src/views/settings/components/privacy/presets.ts new file mode 100644 index 000000000..d2e7ed150 --- /dev/null +++ b/frontend/src/views/settings/components/privacy/presets.ts @@ -0,0 +1,120 @@ +import type { PrivacyAction, PrivacyRule, PrivacySample, PrivacyScope } from '@/api/modules/settings/privacy' + +export const scopes: { value: PrivacyScope, label: string }[] = [ + { value: 'turn_metadata', label: '工作区附加信息' }, + { value: 'desktop_git_context', label: 'Desktop Git 信息' }, + { value: 'environment_text', label: '自动环境上下文' }, + { value: 'request_body', label: '指定请求字段' }, + { value: 'request_header', label: '指定请求头' }, +] +export const actions: { value: PrivacyAction, label: string }[] = [ + { value: 'regex_replace', label: '正则替换' }, + { value: 'set_value', label: '固定值' }, + { value: 'rename_key', label: '字段名替换' }, + { value: 'remove_field', label: '删除字段 / 数组项' }, +] +export const fieldsByScope: Partial> = { + turn_metadata: [ + { value: '', label: '自定义字段路径' }, + { value: '$.workspaces.*.associated_remote_urls.*', label: 'Git 远端地址' }, + { value: '$.workspaces', label: '工作区路径键' }, + { value: '$.workspaces.*.latest_git_commit_hash', label: 'Git 提交标识' }, + ], + desktop_git_context: [ + { value: '', label: '自定义字段路径' }, + { value: '$.remotes', label: 'Git 远端集合' }, + { value: '$.remotes[*].fetchUrl', label: 'Git 远端地址' }, + { value: '$.headCommitSha', label: 'Git 提交标识' }, + { value: '$.worktreeId', label: '工作区指纹' }, + { value: '$.appVersion', label: '应用版本' }, + { value: '$.appServerVersion', label: 'Core 版本' }, + ], +} + +export function newRule(): PrivacyRule { + return { id: Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join(''), name: '', enabled: false, scope: 'turn_metadata', selector: '$.workspaces.*.associated_remote_urls.*', action: 'regex_replace', pattern: '', replacement: '', value: '', replaceAll: true, caseInsensitive: false, multiLine: false } +} + +export const presets: { name: string, values: Partial }[] = [ + { name: '工作区仓库地址脱敏', values: { scope: 'turn_metadata', selector: '$.workspaces.*.associated_remote_urls.*', action: 'regex_replace', pattern: '(?s)^.+$', replacement: '[REDACTED]' } }, + { name: '工作区用户名替换', values: { scope: 'turn_metadata', selector: '$.workspaces', action: 'rename_key', pattern: String.raw`(?i)^((?:[a-z]:)?[/\\](?:home|users)[/\\])[^/\\]+`, replacement: `\${1}user` } }, + { name: '移除工作区提交标识', values: { scope: 'turn_metadata', selector: '$.workspaces.*.latest_git_commit_hash', action: 'remove_field', pattern: null } }, + // Desktop Git 的这些字段可能为 null，按字段移除，避免正则因类型不匹配撤销整条规则 + // remotes 本身也可能为 null，预设处理整个远端集合，单个地址仍可从常用字段选择 + { name: '移除 Desktop Git 远端', values: { scope: 'desktop_git_context', selector: '$.remotes', action: 'remove_field', pattern: null } }, + { name: '移除 Desktop 提交标识', values: { scope: 'desktop_git_context', selector: '$.headCommitSha', action: 'remove_field', pattern: null } }, + { name: '移除 Desktop 工作区指纹', values: { scope: 'desktop_git_context', selector: '$.worktreeId', action: 'remove_field', pattern: null } }, + { name: '移除 Desktop 应用版本', values: { scope: 'desktop_git_context', selector: '$.appVersion', action: 'remove_field', pattern: null } }, + { name: '移除 Desktop Core 版本', values: { scope: 'desktop_git_context', selector: '$.appServerVersion', action: 'remove_field', pattern: null } }, +] + +// 样本使用虚构仓库、路径与标识，字段形状对应官方 Core 和 Desktop Git 上下文 +const metadata = { workspaces: { '/home/alex/projects/demo': { associated_remote_urls: { origin: 'https://git.example.invalid/private-org/repo.git', public: 'git@git.example.invalid:public/repo.git' }, latest_git_commit_hash: 'a'.repeat(40), has_changes: true } }, request_kind: 'turn' } +const desktopGit = { + schemaVersion: 1, + phase: 'before_turn_request', + hostId: 'local', + sessionId: 'example-session', + threadId: 'example-thread', + clientUserMessageId: 'example-message', + appVersion: '26.1007.21434', + appServerVersion: '0.162.0-alpha.17.2', + status: 'partial', + capturedAt: '2026-10-09T12:00:00Z', + worktreeId: 'b'.repeat(40), + headCommitSha: 'a'.repeat(40), + branch: 'main', + detached: false, + upstream: { remoteName: 'origin', mergeRef: 'refs/heads/main', trackingRef: 'refs/remotes/origin/main' }, + remotes: [ + { name: 'origin', fetchUrl: 'https://git.example.invalid/private-org/repo.git' }, + { name: 'upstream', fetchUrl: 'ssh://git.example.invalid/public/repo.git' }, + { name: 'unavailable', fetchUrl: null, unavailableReason: 'unsafe_url' }, + ], + remotesOmitted: 0, +} + +function desktopContext(value: unknown): string { + const text = JSON.stringify(value).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>') + return `${text}` +} + +function contextMessage(text: string) { + return { role: 'user', content: [{ type: 'input_text', text }] } +} + +export function sampleText(scope: PrivacyScope): string { + if (scope === 'turn_metadata') + return JSON.stringify(metadata, null, 2) + if (scope === 'desktop_git_context') + return JSON.stringify(desktopGit, null, 2) + if (scope === 'environment_text') + return '\n/home/alex/projects/demo\nAsia/Shanghai\n' + if (scope === 'request_header') + return 'example-client' + return JSON.stringify({ metadata: { workspace: '/home/alex/projects/demo' } }, null, 2) +} + +export function makeSample(scope: PrivacyScope, text: string, selector = ''): PrivacySample { + const sample: PrivacySample = { body: {}, headers: {}, turnMetadata: null } + if (scope === 'turn_metadata') { + sample.turnMetadata = JSON.stringify(JSON.parse(text)) + } + else if (scope === 'request_header') { + sample.headers[selector] = [text] + } + else if (scope === 'request_body') { + sample.body = JSON.parse(text) + } + else { + const context = scope === 'desktop_git_context' + ? desktopContext(JSON.parse(text)) + : text + sample.body = { input: [contextMessage(context)] } + } + return sample +} + +export function exampleRequest(): PrivacySample { + return { body: { input: [contextMessage(desktopContext(desktopGit))], client_metadata: { 'x-codex-turn-metadata': JSON.stringify(metadata) } }, headers: { 'x-codex-turn-metadata': [JSON.stringify(metadata)] }, turnMetadata: null } +} diff --git a/frontend/src/views/settings/composables/usePrivacyPreview.ts b/frontend/src/views/settings/composables/usePrivacyPreview.ts new file mode 100644 index 000000000..d121dea3c --- /dev/null +++ b/frontend/src/views/settings/composables/usePrivacyPreview.ts @@ -0,0 +1,28 @@ +import type { CodexPrivacyPolicy, PrivacyPreviewResult, PrivacySample } from '@/api/modules/settings/privacy' +import { shallowRef } from 'vue' +import { previewPrivacyPolicy } from '@/api/modules/settings/privacy' +import { useRequestState } from '@/composables/useRequestState' + +export function usePrivacyPreview() { + const state = useRequestState() + const result = shallowRef() + function invalidate() { + state.invalidate() + result.value = undefined + state.error.value = '' + } + async function run(policy: CodexPrivacyPolicy, sample: PrivacySample) { + const id = state.start() + result.value = undefined + try { + const response = await previewPrivacyPolicy({ policy, ...sample }, { signal: state.signal }) + if (!state.isCurrent(id)) + return undefined + result.value = response + return response + } + catch (cause) { state.fail(id, cause) } + finally { state.finish(id) } + } + return { loading: state.loading, error: state.error, result, run, invalidate } +} diff --git a/frontend/src/views/settings/composables/useSettingsForm.ts b/frontend/src/views/settings/composables/useSettingsForm.ts index 70818c705..e9453643c 100644 --- a/frontend/src/views/settings/composables/useSettingsForm.ts +++ b/frontend/src/views/settings/composables/useSettingsForm.ts @@ -1,5 +1,5 @@ import type { rotationOptions } from '../constants' -import type { AccountAffinity, SmartSchedulingConfig } from '@/api' +import type { AccountAffinity, CodexPrivacyPolicy, SmartSchedulingConfig } from '@/api' import type { ProviderRequestProfiles, ProviderRequestProfileUpdates } from '@/api/modules/settings/profiles' import { toast } from '@codex-proxy/ui' import { cloneDeep, isEqual } from 'es-toolkit' @@ -26,6 +26,7 @@ export function useSettingsForm() { configRevision: 0, smartScheduling: undefined as SmartSchedulingConfig | undefined, providerRequestProfiles: {} as ProviderRequestProfiles, + codexPrivacyPolicy: { enabled: false, onError: 'skip_rule', rules: [] } as CodexPrivacyPolicy, requestLocationEnabled: false, requestLocation: { country: '', region: '', city: '', timezone: '' }, refreshMarginSeconds: null as number | null, @@ -114,6 +115,7 @@ export function useSettingsForm() { } function applySettings(data: Awaited>) { + form.codexPrivacyPolicy = cloneDeep(data.codexPrivacyPolicy) form.configRevision = data.configRevision form.requestLocationEnabled = data.requestLocationEnabled form.requestLocation = { ...data.requestLocation } @@ -290,6 +292,7 @@ export function useSettingsForm() { savedSettings.form.providerRequestProfiles, form.providerRequestProfiles, ), + codexPrivacyPolicy: cloneDeep(form.codexPrivacyPolicy), requestLocationEnabled: form.requestLocationEnabled, requestLocation, modelMappings: mappingPayload(), diff --git a/frontend/src/views/settings/index.vue b/frontend/src/views/settings/index.vue index b8090ac56..62b8ce3fd 100644 --- a/frontend/src/views/settings/index.vue +++ b/frontend/src/views/settings/index.vue @@ -12,6 +12,7 @@ import SettingsBackupSection from './components/backup/SettingsBackupSection.vue import ClientProfileCard from './components/ClientProfileCard.vue' import ConcurrencyPolicyCard from './components/ConcurrencyPolicyCard.vue' import ModelAliasesCard from './components/ModelAliasesCard.vue' +import PrivacyPolicyCard from './components/privacy/PrivacyPolicyCard.vue' import RequestLocationCard from './components/RequestLocationCard.vue' import RotationStrategyCard from './components/RotationStrategyCard.vue' import SettingsAccessSection from './components/SettingsAccessSection.vue' @@ -192,6 +193,7 @@ watch(section, (value) => { :disabled="disabled" /> +

diff --git a/frontend/src/views/settings/pricing/PricingUnit.vue b/frontend/src/views/settings/pricing/PricingUnit.vue index 179574992..25f93e0e1 100644 --- a/frontend/src/views/settings/pricing/PricingUnit.vue +++ b/frontend/src/views/settings/pricing/PricingUnit.vue @@ -1,5 +1,5 @@