From f7f59103a310166101de56bc21e85e8501586d07 Mon Sep 17 00:00:00 2001 From: Zyy Date: Thu, 22 Oct 2026 22:06:01 +0800 Subject: [PATCH] fix: harden request lifecycles and align repository documentation Preserve request evidence and cold middleware boundaries across cancellation. Make backup auditing atomic, retain committed account-group results, and serialize restart operations. Unify frontend stream authentication and session recovery, clear stale secrets, and remove redundant runtime paths. Correct CI permissions and database test URL handling. Align documentation with current module responsibilities, APIs, and validation workflows, including both submodules. --- .../references/implementation.md | 2 +- .../cpr-release/references/operations.md | 4 +- .../cpr-release/references/preparation.md | 2 +- .github/workflows/ci.yml | 3 + CONTRIBUTING.md | 3 + README.md | 3 +- backend/apps/plugin-cli/README.md | 11 +- .../crates/gateway-admin/src/backup/task.rs | 4 +- .../crates/gateway-admin/src/model/audit.rs | 8 + .../crates/gateway-admin/src/ports/backup.rs | 13 +- .../src/use_case/account_groups.rs | 10 +- .../gateway-admin/src/use_case/backup.rs | 64 +- .../gateway-admin/tests/backup/support.rs | 29 +- .../crates/gateway-admin/tests/backup/task.rs | 22 +- .../tests/use_case/account_groups.rs | 70 ++- .../gateway-admin/tests/use_case/backup.rs | 14 +- .../gateway-admin/tests/use_case/mod.rs | 9 +- .../src/admin/plugins/management.rs | 19 +- .../gateway-api/src/openai/responses/http.rs | 38 +- .../tests/admin/plugins/management.rs | 146 ++++- .../gateway-core/src/engine/coordinator.rs | 582 +++++++++++------- .../gateway-core/src/engine/provider.rs | 181 ++++-- .../gateway-core/tests/engine/coordinator.rs | 195 +++++- .../tests/engine/middleware/cold.rs | 206 +++++++ .../tests/engine/middleware/mod.rs | 2 + .../src/system_update/operations.rs | 50 +- .../tests/system_update/operations.rs | 107 ++++ .../src/adapter/command_line/execution.rs | 8 +- .../src/adapter/frontend_authentication.rs | 8 +- .../src/adapter/management/callback.rs | 6 +- .../runtime/src/adapter/management/mod.rs | 4 +- .../runtime/src/adapter/middleware/http.rs | 31 +- .../runtime/src/adapter/middleware/request.rs | 69 +-- .../src/adapter/middleware/websocket.rs | 6 +- .../src/adapter/upstream_adapter/event.rs | 31 +- .../src/adapter/upstream_adapter/execution.rs | 33 +- .../gateway-plugin/runtime/src/rpc/session.rs | 5 - backend/crates/gateway-plugin/sdk/README.md | 39 +- .../gateway-plugin/sdk/docs/capabilities.md | 14 +- .../gateway-plugin/sdk/docs/manifest.md | 26 +- .../sdk/docs/upstream-adapters.md | 2 + .../gateway-plugin/sdk/docs/usage-data.md | 4 +- .../src/postgres/account_groups.rs | 17 +- .../gateway-store/src/postgres/backup.rs | 84 ++- backend/crates/gateway-store/tests/bundle.rs | 11 +- .../tests/postgres/account_groups.rs | 85 +++ .../gateway-store/tests/postgres/backup.rs | 133 +++- .../gateway-store/tests/postgres/pricing.rs | 9 +- .../gateway-store/tests/redis/worker_lease.rs | 14 +- backend/crates/gateway-store/tests/support.rs | 10 + backend/migrations/README.md | 10 +- deploy/README.md | 15 +- docs/api.md | 90 +-- docs/architecture.md | 95 +-- docs/development.md | 3 + docs/plugins.md | 16 +- docs/theme.md | 13 +- frontend/src/api/modules/keys.ts | 3 +- .../src/api/modules/plugins/extensions.ts | 25 +- frontend/src/api/request/index.ts | 18 +- frontend/src/api/request/plugins/index.ts | 1 + frontend/src/api/request/plugins/raw.ts | 52 +- frontend/src/api/request/plugins/session.ts | 11 +- .../views/keys/composables/useApiKeyUse.ts | 64 +- .../plugins/components/PluginEnableModal.vue | 4 +- .../plugins/components/PluginManagement.vue | 1 + modules/plugins | 2 +- modules/ui | 2 +- 68 files changed, 2061 insertions(+), 810 deletions(-) create mode 100644 backend/crates/gateway-core/tests/engine/middleware/cold.rs diff --git a/.agents/skills/cpr-plugin-dev/references/implementation.md b/.agents/skills/cpr-plugin-dev/references/implementation.md index ce18296db..dce965823 100644 --- a/.agents/skills/cpr-plugin-dev/references/implementation.md +++ b/.agents/skills/cpr-plugin-dev/references/implementation.md @@ -6,7 +6,7 @@ ## 清单与处理器 - `publisher` 与机器短名 `name` 派生插件 ID，`displayName` 用于展示；复制示例后同时调整扩展项 ID 与注册结果 -- `contributes` 每种能力最多一项，普通贡献项可省略派生 ID、能力版本和固定阶段；中间件必须声明版本 `4` 并显式选择阶段，上游适配器必须声明版本 `2`。自定义完整扩展项 ID 属于自己的插件命名空间，运行注册与规范化清单一致 +- `contributes` 每种能力最多一项，派生 ID 和固定阶段可省略；能力版本与中间件阶段按[扩展项简写](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)声明。自定义完整扩展项 ID 属于自己的插件命名空间，运行注册与规范化清单一致 - 作者清单用 `Manifest::from_author_slice` 规范化，不手写 `package`、固定阶段或重复注册；CLI 生成平台、协议与资源摘要，版本范围不能使用全版本通配 - 安装后宿主会按 `configurationSchema` 准备默认配置，配置完整即可启用；默认绑定的空范围不限制请求。需要业务参数时声明真实必填项，通过 `secretFields` 声明敏感字段，不依赖用户再走一遍自建安装向导 - Rust 插件只依赖公开 `gateway-plugin-sdk`；需要异步会话辅助时开启 `io`，使用 `PluginSession` 管理握手、回调关联、流控、取消和关闭 diff --git a/.agents/skills/cpr-release/references/operations.md b/.agents/skills/cpr-release/references/operations.md index f3afc7dd2..74b14d2d8 100644 --- a/.agents/skills/cpr-release/references/operations.md +++ b/.agents/skills/cpr-release/references/operations.md @@ -20,7 +20,7 @@ 调用入口前重新核对当前脚本的副作用、目标版本和待推送提交，保留无关工作区内容，不自动清理或暂存用户文件。 先编辑 `release/notes.md`，保留为未提交改动（已暂存或未暂存均可）；其他已跟踪文件的改动应先单独提交。 -脚本会更新 `release/version.yaml`，将它与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。 +脚本会更新 `release/version.yaml`，按[接口弃用规则](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)校验并维护 `plugin-api-deprecations.json`，将这些差异与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。 不要提前单独提交说明或把版本文件改成目标版本；脚本要求说明相对 HEAD 有改动，并自行更新版本号。 本任务需要的提交按[项目约定](../../../../CONTRIBUTING.md#项目约定)保留英文 Conventional Commits 与相应署名。 @@ -30,7 +30,7 @@ release/publish "$release_version" ``` -该命令会创建包含版本号和发布说明的单次提交与 annotated tag，原子推送分支和 tag，再从长期分支手动触发 `release.yml`。 +该命令会创建上述版本提交与 annotated tag，原子推送分支和 tag，再从长期分支手动触发 `release.yml`。 它没有只读预演模式；不能为了检查或演练而执行。发布工作流读取 tag 中的说明，修改 tag 之后的本地文件不会改变该次构建。 只调整已发布页面的标题或正文时，编辑对应 GitHub Release 的 `name` 或 `body` 并回读核对； diff --git a/.agents/skills/cpr-release/references/preparation.md b/.agents/skills/cpr-release/references/preparation.md index b1add4142..7a4155d44 100644 --- a/.agents/skills/cpr-release/references/preparation.md +++ b/.agents/skills/cpr-release/references/preparation.md @@ -19,7 +19,7 @@ - 只翻译已有准确说明时保持事实与范围，不混入其他变更 - 常规文档仍只描述当前状态；必要修订执行[文档检查](../../cpr-dev-guide/references/documentation.md)，不复制发布说明中的版本变化列表 -实际发布由 `release/publish` 提交说明与版本文件，准备说明时不要提前单独提交或把版本文件改成目标版本 +实际发布由 `release/publish` 提交说明、版本文件及[插件接口弃用清单](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)的必要更新，准备说明时不要提前单独提交或把版本文件改成目标版本 用户只要求准备时保留本地改动，纯说明修改无需重建前后端 ## 发行准备检查 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c6d0ace9..a1ff68ac6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: changes: runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read + pull-requests: read outputs: backend: ${{ steps.filter.outputs.backend }} frontend: ${{ steps.filter.outputs.frontend }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0041c8a28..f4aa76ea4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,6 +149,9 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名： | [插件使用](docs/plugins.md) | 如何安装、授权、配置和使用插件 | 管理员操作与影响，不展开 SDK 调用或宿主事务实现 | | [插件 SDK](backend/crates/gateway-plugin/sdk/README.md) / [CLI](backend/apps/plugin-cli/README.md) | 如何编写和打包插件 | 作者合同、权限边界、示例与打包验证 | | [数据库迁移](backend/migrations/README.md) | 如何维护 schema 和准备测试库 | 冻结规则、升级约束和测试环境，不罗列迁移历史 | +| [共用组件](modules/ui/README.md)及其组件参考 | 如何安装、使用和开发独立 UI 库 | 组件公开 API、主题能力与库内开发；宿主页面约定归管理端主题，跨仓库联调归开发文档 | +| [官方插件示例](modules/plugins/README.md)及其示例说明 | 如何构建、使用和维护示例插件 | 示例自身的功能、配置与验证；通用协议归 SDK，宿主安装操作归插件使用文档 | +| [AGENTS.md](AGENTS.md) 与[项目技能](.agents/skills/cpr-dev-guide/SKILL.md) | AI 如何选择并执行任务流程 | 任务匹配、执行步骤和规范入口，不另建模块合同、版本表或协作标准 | | 本文 | 如何协作、审查和交付 | 流程、判断标准与验证要求，任务入口由 [开发技能](.agents/skills/cpr-dev-guide/SKILL.md) 串联 | - 同一事实只在所属文档完整说明，其他位置按读者任务保留必要摘要和链接 diff --git a/README.md b/README.md index d0502d011..512c435a3 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,8 @@ curl http://127.0.0.1:8080/v1/models \ | 部署与使用 | [部署、备份与恢复](deploy/README.md) · [客户端接入与生图](deploy/README.md#客户端配置) | | 接口集成 | [API 参考](docs/api.md) · [模型定价](docs/api.md#模型定价) | | 使用插件 | [安装、配置与使用](docs/plugins.md) | -| 开发插件 | [SDK 与合同](backend/crates/gateway-plugin/sdk/README.md) · [打包工具](backend/apps/plugin-cli/README.md) | +| 开发插件 | [SDK 与合同](backend/crates/gateway-plugin/sdk/README.md) · [打包工具](backend/apps/plugin-cli/README.md) · [官方示例](modules/plugins/README.md) | +| 开发共用组件 | [组件库说明](modules/ui/README.md) · [组件参考](modules/ui/playground/components/index.md) | | 开发宿主 | [开发与源码联调](docs/development.md) · [系统架构](docs/architecture.md) · [管理端主题](docs/theme.md) · [数据库迁移](backend/migrations/README.md) | | 参与协作 | [贡献、审查与验证](CONTRIBUTING.md) | diff --git a/backend/apps/plugin-cli/README.md b/backend/apps/plugin-cli/README.md index 0fd039619..4aef81cee 100644 --- a/backend/apps/plugin-cli/README.md +++ b/backend/apps/plugin-cli/README.md @@ -26,10 +26,15 @@ cpr-plugin package \ --output-dir /path/to/output ``` -资源映射相对于作者清单所在目录解析。输出包括完整插件 ID、版本和目标 triple 命名的 `.tar.gz` 及 `.sha256`；归档根目录直接包含 `plugin.json`、可执行文件和已声明资源，不包含开发依赖 +`--resource-map web=web/dist` 将包内 `web/` 前缀映射到作者清单所在目录下的 `web/dist/`。 +可重复传入不同前缀，源目录必须位于项目内；没有对应资源时省略该参数，未使用的映射会被拒绝。 +未映射的资源按清单中的相对路径读取，只收集 `resources` 声明的文件 -作者清单的省略规则、能力版本和挂载阶段见 [SDK 扩展项声明](../../crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)。 -CLI 与 SDK 的 `Manifest::from_author_slice`、`PluginBuilder::from_json` 共用规范化入口，归档只保留补全字段后的 `plugin.json` +输出为 `<插件 ID>-<版本>-<目标 triple>.tar.gz` 及同名 `.tar.gz.sha256`；归档根目录直接包含 `plugin.json`、可执行文件和声明资源。 +输入清单及生成后的清单均不得超过 64 KiB，压缩包上限为 32 MiB，展开归档上限为 128 MiB + +作者清单不填写 `package`；字段省略规则、能力版本和挂载阶段见 [SDK 扩展项声明](../../crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)。 +CLI 生成规范化后的 `plugin.json`，补入协议版本、目标平台和文件摘要。打包成功只证明输入符合包合同，实际启动与能力仍需在目标宿主验证 支持 `x86_64-unknown-linux-gnu`、`aarch64-unknown-linux-gnu` 和 `aarch64-apple-darwin`。 完整开发示例在独立仓库 `codex-proxy-plugins` 的 `examples/workbench`，可通过[子模块](../../../docs/development.md#源码联调)检出；本工具不负责源码构建、上传或启用插件 diff --git a/backend/crates/gateway-admin/src/backup/task.rs b/backend/crates/gateway-admin/src/backup/task.rs index 457149746..85284fa85 100644 --- a/backend/crates/gateway-admin/src/backup/task.rs +++ b/backend/crates/gateway-admin/src/backup/task.rs @@ -649,7 +649,7 @@ impl BackupTask { for record in expired { let Some(record) = self .repository - .transition_to_deleting(&record.id, now) + .transition_to_deleting(&record.id, now, None) .await .map_err(repo_error)? else { @@ -673,7 +673,7 @@ impl BackupTask { for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) { let Some(record) = self .repository - .transition_to_deleting(&decision.record_id, now) + .transition_to_deleting(&decision.record_id, now, None) .await .map_err(repo_error)? else { diff --git a/backend/crates/gateway-admin/src/model/audit.rs b/backend/crates/gateway-admin/src/model/audit.rs index eb924a6d5..3ccf6aacb 100644 --- a/backend/crates/gateway-admin/src/model/audit.rs +++ b/backend/crates/gateway-admin/src/model/audit.rs @@ -19,6 +19,10 @@ pub enum MutationAuditOperation { ProviderAccountUpdate, BackupStorageUpdate, BackupScheduleUpdate, + BackupCreate, + BackupDelete, + BackupConnectionTest, + BackupDownloadUrl, ProviderAccountAdaptConcurrency, ProviderAccountRecover, ProviderAccountBatchUpdate, @@ -76,6 +80,10 @@ impl MutationAuditOperation { Self::ProviderAccountUpdate => ("update", "provider_account"), Self::BackupStorageUpdate => ("backup.s3_config_updated", "backup_settings"), Self::BackupScheduleUpdate => ("backup.schedule_updated", "backup_settings"), + Self::BackupCreate => ("backup.created", "backup"), + Self::BackupDelete => ("backup.delete_requested", "backup"), + Self::BackupConnectionTest => ("backup.s3_connection_tested", "backup_settings"), + Self::BackupDownloadUrl => ("backup.download_url_created", "backup"), Self::ProviderAccountAdaptConcurrency => ("adapt_concurrency", "provider_account"), Self::ProviderAccountRecover => ("recover", "provider_account"), Self::ProviderAccountBatchUpdate => ("batch_update", "provider_account"), diff --git a/backend/crates/gateway-admin/src/ports/backup.rs b/backend/crates/gateway-admin/src/ports/backup.rs index b32c42fbf..7444af434 100644 --- a/backend/crates/gateway-admin/src/ports/backup.rs +++ b/backend/crates/gateway-admin/src/ports/backup.rs @@ -66,8 +66,12 @@ pub trait BackupRepository: Send + Sync { at: DateTime, ) -> AdminStoreResult; - /// 插入 queued 任务；存在活跃任务时返回 `backup.active_task_conflict` - async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult; + /// 同事务插入人工 queued 任务与审计；存在活跃任务时返回 `backup.active_task_conflict` + async fn insert_backup_record( + &self, + seed: BackupRecordSeed, + context: &MutationContext, + ) -> AdminStoreResult; /// 核对启停、Cron、时区及旧游标后，同事务推进游标并插入 scheduled 任务 /// 配置已变化时不写入；同一时间点或活跃冲突时只推进游标，返回 `false` @@ -111,11 +115,12 @@ pub trait BackupRepository: Send + Sync { now: DateTime, ) -> AdminStoreResult>; - /// 迁移到删除流程（completed/failed → deleting）；供管理员与保留策略调用 + /// 迁移到删除流程（completed/failed → deleting）；人工操作同事务记录审计，保留策略传 None async fn transition_to_deleting( &self, id: &str, now: DateTime, + context: Option<&MutationContext>, ) -> AdminStoreResult>; /// 远端删除成功后硬删除记录行 @@ -313,6 +318,7 @@ impl BackupRepository for UnavailableBackupRepository { async fn insert_backup_record( &self, _seed: BackupRecordSeed, + _context: &MutationContext, ) -> AdminStoreResult { Err(disabled_store()) } @@ -363,6 +369,7 @@ impl BackupRepository for UnavailableBackupRepository { &self, _id: &str, _now: DateTime, + _context: Option<&MutationContext>, ) -> AdminStoreResult> { Err(disabled_store()) } diff --git a/backend/crates/gateway-admin/src/use_case/account_groups.rs b/backend/crates/gateway-admin/src/use_case/account_groups.rs index 381bcf3b1..4bfe9354c 100644 --- a/backend/crates/gateway-admin/src/use_case/account_groups.rs +++ b/backend/crates/gateway-admin/src/use_case/account_groups.rs @@ -114,10 +114,14 @@ impl DefaultAccountGroupService { result: Result, ) -> Result { let mut result = result.map_err(|error| map_store_error(error, "account group"))?; - if let Some(record) = result.record.as_mut() { - self.enrich_records(std::slice::from_mut(record)).await?; - } publish_committed(self.snapshot.as_ref(), result.config_revision).await?; + if let Some(record) = result.record.as_mut() + && let Err(error) = self.enrich_records(std::slice::from_mut(record)).await + { + tracing::warn!(group_id = %result.id.as_str(), error = %error, "committed account group runtime projection is unavailable"); + // record 可缺省；不要用未完成的容量投影冒充有效展示数据。 + result.record = None; + } Ok(result) } } diff --git a/backend/crates/gateway-admin/src/use_case/backup.rs b/backend/crates/gateway-admin/src/use_case/backup.rs index c43f5e91b..7756fcb67 100644 --- a/backend/crates/gateway-admin/src/use_case/backup.rs +++ b/backend/crates/gateway-admin/src/use_case/backup.rs @@ -7,11 +7,10 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use gateway_core::runtime::SnapshotControl; use secrecy::ExposeSecret as _; -use uuid::Uuid; use crate::model::{ AdminError, AdminErrorKind, MutationContext, - auth::{AdminAuditEvent, AuditActorKind}, + audit::{MutationAuditIntent, MutationAuditOperation}, backup::{ BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult, @@ -180,8 +179,8 @@ impl BackupService for DefaultBackupService { } self.append_audit( context, - "backup.s3_connection_tested", - "backup_settings", + MutationAuditOperation::BackupConnectionTest, + "1", vec![], ) .await?; @@ -219,11 +218,9 @@ impl BackupService for DefaultBackupService { .map_err(map_backup_error)?; let record = self .repository - .insert_backup_record(seed) + .insert_backup_record(seed, context) .await .map_err(|error| map_store_error(error, "backup record"))?; - self.append_audit(context, "backup.created", "backup", vec![record.id.clone()]) - .await?; Ok(record) } @@ -264,9 +261,9 @@ impl BackupService for DefaultBackupService { .map_err(map_backup_error)?; self.append_audit( context, - "backup.download_url_created", - "backup", - vec![backup_id.to_owned()], + MutationAuditOperation::BackupDownloadUrl, + backup_id, + vec![], ) .await?; Ok(DownloadUrlResult { @@ -290,17 +287,10 @@ impl BackupService for DefaultBackupService { let now = Utc::now(); let deleting = self .repository - .transition_to_deleting(backup_id, now) + .transition_to_deleting(backup_id, now, Some(context)) .await .map_err(|error| map_store_error(error, "backup record"))? .ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?; - self.append_audit( - context, - "backup.delete_requested", - "backup", - vec![backup_id.to_owned()], - ) - .await?; Ok(deleting) } } @@ -317,41 +307,15 @@ impl DefaultBackupService { async fn append_audit( &self, context: &MutationContext, - action: &str, + operation: MutationAuditOperation, entity_ref: &str, changed_fields: Vec, ) -> Result<(), AdminError> { - let actor_kind = match context.actor { - crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession, - crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey, - crate::model::MutationActor::System => AuditActorKind::System, - }; - let actor_ref = match &context.actor { - crate::model::MutationActor::AdminSession { admin_user_id } => { - crate::model::auth::admin_session_actor_ref(admin_user_id) - } - crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(), - crate::model::MutationActor::System => "system".to_owned(), - }; - let actor_admin_user_id = match &context.actor { - crate::model::MutationActor::AdminSession { admin_user_id } => { - Some(admin_user_id.clone()) - } - _ => None, - }; - let event = AdminAuditEvent { - id: format!("audit_{}", Uuid::now_v7().simple()), - actor_kind, - actor_admin_user_id, - actor_ref, - request_id: Some(context.request_id.clone()), - action: action.to_owned(), - entity_kind: "backup".to_owned(), - entity_ref: entity_ref.to_owned(), - config_revision: None, - changed_fields, - occurred_at: Utc::now(), - }; + let event = MutationAuditIntent { + operation, + entity_ref, + } + .event(context, changed_fields); self.auth .append_audit_event(event) .await diff --git a/backend/crates/gateway-admin/tests/backup/support.rs b/backend/crates/gateway-admin/tests/backup/support.rs index 9510999e1..1c5b0c177 100644 --- a/backend/crates/gateway-admin/tests/backup/support.rs +++ b/backend/crates/gateway-admin/tests/backup/support.rs @@ -92,6 +92,7 @@ pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord { /// 内存版备份仓储 pub(crate) struct FakeBackupRepository { + audits: Mutex>, settings: Mutex, settings_unavailable: AtomicBool, records: Mutex>, @@ -101,11 +102,16 @@ impl FakeBackupRepository { pub(crate) fn new(settings: BackupSettings) -> Self { Self { settings: Mutex::new(settings), + audits: Mutex::new(Vec::new()), settings_unavailable: AtomicBool::new(false), records: Mutex::new(Vec::new()), } } + pub(crate) fn audit_events(&self) -> Vec { + self.audits.lock().expect("audits").clone() + } + pub(crate) fn set_settings_unavailable(&self, unavailable: bool) { self.settings_unavailable .store(unavailable, Ordering::SeqCst); @@ -207,7 +213,11 @@ impl BackupRepository for FakeBackupRepository { Ok(true) } - async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + async fn insert_backup_record( + &self, + seed: BackupRecordSeed, + context: &MutationContext, + ) -> AdminStoreResult { let mut records = self.records.lock().expect("records"); if records.iter().any(|record| record.status.is_active()) { return Err(AdminStoreError::new( @@ -218,6 +228,13 @@ impl BackupRepository for FakeBackupRepository { } let record = queued_record(&seed); records.push(record.clone()); + self.audits.lock().expect("audits").push( + gateway_admin::model::audit::MutationAuditIntent { + operation: gateway_admin::model::audit::MutationAuditOperation::BackupCreate, + entity_ref: &record.id, + } + .event(context, vec!["status".to_owned()]), + ); Ok(record) } @@ -401,6 +418,7 @@ impl BackupRepository for FakeBackupRepository { &self, id: &str, now: DateTime, + context: Option<&MutationContext>, ) -> AdminStoreResult> { let mut records = self.records.lock().expect("records"); let Some(record) = records @@ -411,6 +429,15 @@ impl BackupRepository for FakeBackupRepository { }; record.status = BackupStatus::Deleting; record.updated_at = now; + if let Some(context) = context { + self.audits.lock().expect("audits").push( + gateway_admin::model::audit::MutationAuditIntent { + operation: gateway_admin::model::audit::MutationAuditOperation::BackupDelete, + entity_ref: id, + } + .event(context, vec!["status".to_owned()]), + ); + } Ok(Some(record.clone())) } diff --git a/backend/crates/gateway-admin/tests/backup/task.rs b/backend/crates/gateway-admin/tests/backup/task.rs index 60a1d1182..ed2f809f9 100644 --- a/backend/crates/gateway-admin/tests/backup/task.rs +++ b/backend/crates/gateway-admin/tests/backup/task.rs @@ -32,7 +32,7 @@ async fn backup_task_runs_full_dump_upload_verify_pipeline() { expires_at: None, }; repository - .insert_backup_record(seed.clone()) + .insert_backup_record(seed.clone(), &system_context()) .await .expect("insert queued"); @@ -73,7 +73,7 @@ async fn backup_task_marks_failed_when_dump_port_fails() { expires_at: None, }; repository - .insert_backup_record(seed.clone()) + .insert_backup_record(seed.clone(), &system_context()) .await .expect("insert queued"); @@ -105,7 +105,7 @@ async fn backup_task_marks_failed_with_the_upload_error() { expires_at: None, }; repository - .insert_backup_record(seed.clone()) + .insert_backup_record(seed.clone(), &system_context()) .await .expect("insert queued"); @@ -138,7 +138,7 @@ async fn backup_task_recovery_marks_failed_with_the_upload_error() { expires_at: None, }; repository - .insert_backup_record(seed.clone()) + .insert_backup_record(seed.clone(), &system_context()) .await .expect("insert queued"); let cancellation = CancellationToken::new(); @@ -185,14 +185,14 @@ async fn retention_cleans_expired_scheduled_backups() { expires_at: None, }; repository - .insert_backup_record(old_seed.clone()) + .insert_backup_record(old_seed.clone(), &system_context()) .await .expect("insert old"); // 旧备份 40 天前完成（释放活跃名额），再插入新备份 let now = Utc::now(); repository.set_completed(&backup_id("old"), now - Duration::days(40)); repository - .insert_backup_record(new_seed.clone()) + .insert_backup_record(new_seed.clone(), &system_context()) .await .expect("insert new"); // 新备份 1 天前完成 @@ -279,7 +279,10 @@ async fn completed_recovery_removes_staging_for_dumping_and_uploading_tasks() { object_key: "recovery.dump".to_owned(), expires_at: None, }; - repository.insert_backup_record(seed.clone()).await.unwrap(); + repository + .insert_backup_record(seed.clone(), &system_context()) + .await + .unwrap(); let artifact = dump .dump(DumpRequest { backup_id: seed.id.clone(), @@ -344,7 +347,10 @@ async fn manual_backup_preserves_primary_and_final_delete_failures() { object_key: "codex/cleanup-failure.dump".into(), expires_at: expiring.then(|| Utc::now() + Duration::hours(1)), }; - repository.insert_backup_record(seed.clone()).await.unwrap(); + repository + .insert_backup_record(seed.clone(), &system_context()) + .await + .unwrap(); let task = BackupTask::new(repository.clone(), dump.clone(), store.clone()); task.run_cycle(&CancellationToken::new()).await.unwrap(); let records = repository.all_records(); diff --git a/backend/crates/gateway-admin/tests/use_case/account_groups.rs b/backend/crates/gateway-admin/tests/use_case/account_groups.rs index 452d0137c..009513163 100644 --- a/backend/crates/gateway-admin/tests/use_case/account_groups.rs +++ b/backend/crates/gateway-admin/tests/use_case/account_groups.rs @@ -149,6 +149,9 @@ async fn group_options_query_should_preserve_store_failures() { #[derive(Default)] struct FakeGroupStore { + projection_failure: bool, + missing_record: bool, + published_revision: Option>, requested_groups: Mutex>, members: Option>, } @@ -223,6 +226,16 @@ impl AccountGroupStore for FakeGroupStore { &self, group_ids: &[AccountGroupId], ) -> AdminStoreResult> { + if self.projection_failure { + assert_eq!( + self.published_revision + .as_ref() + .expect("publication counter") + .load(std::sync::atomic::Ordering::SeqCst), + 2 + ); + return Err(unused()); + } *self.requested_groups.lock().expect("requested groups") = group_ids .iter() .map(|group_id| group_id.as_str().to_owned()) @@ -251,10 +264,16 @@ impl AccountGroupStore for FakeGroupStore { async fn set_account_group_enabled( &self, - _: SetAccountGroupEnabled, + command: SetAccountGroupEnabled, _: &MutationContext, ) -> AdminStoreResult { - Err(unused()) + let mut record = group_record(); + record.enabled = command.enabled; + Ok(AccountGroupMutation { + config_revision: Revision::new(2).expect("committed revision"), + id: record.id.clone(), + record: (!self.missing_record).then_some(record), + }) } async fn delete_account_group( @@ -431,3 +450,50 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() { assert_eq!(created.kind(), AdminErrorKind::Invalid); assert_eq!(updated.kind(), AdminErrorKind::Invalid); } + +struct RecordingPublication(Arc); + +impl gateway_core::runtime::SnapshotControl for RecordingPublication { + fn publish_committed( + &self, + revision: gateway_core::routing::ConfigRevision, + ) -> futures::future::BoxFuture<'_, ()> { + Box::pin(async move { + self.0 + .store(revision.get(), std::sync::atomic::Ordering::SeqCst); + }) + } +} + +#[tokio::test] +async fn committed_group_mutations_publish_even_when_the_display_projection_is_unavailable() { + for missing_record in [false, true] { + let published = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let groups = Arc::new(FakeGroupStore { + projection_failure: true, + missing_record, + published_revision: Some(published.clone()), + ..Default::default() + }); + let services = AdminHarness::new() + .account_groups(groups) + .snapshot(Arc::new(RecordingPublication(published.clone()))) + .build() + .await; + let mutation = services + .account_groups() + .set_enabled( + &crate::backup::support::system_context(), + SetAccountGroupEnabled { + id: group_id(), + enabled: false, + }, + ) + .await + .expect("committed mutation remains successful"); + assert_eq!(mutation.config_revision.get(), 2); + assert_eq!(mutation.id, group_id()); + assert!(mutation.record.is_none()); + assert_eq!(published.load(std::sync::atomic::Ordering::SeqCst), 2); + } +} diff --git a/backend/crates/gateway-admin/tests/use_case/backup.rs b/backend/crates/gateway-admin/tests/use_case/backup.rs index 787fc6a22..a288e8d8d 100644 --- a/backend/crates/gateway-admin/tests/use_case/backup.rs +++ b/backend/crates/gateway-admin/tests/use_case/backup.rs @@ -38,9 +38,12 @@ async fn backup_service_creates_downloads_and_deletes_with_audit() { assert_eq!(created.status, BackupStatus::Queued); assert!(created.object_key.starts_with("codex/")); assert!( - auth.audit_actions() + repository + .audit_events() .iter() - .any(|action| action == "backup.created") + .any(|event| event.action == "backup.created" + && event.entity_ref == created.id + && event.request_id.as_deref() == Some(system_context().request_id.as_str())) ); // 下载地址要求 completed 记录 @@ -65,9 +68,12 @@ async fn backup_service_creates_downloads_and_deletes_with_audit() { .expect("delete backup"); assert_eq!(deleting.status, BackupStatus::Deleting); assert!( - auth.audit_actions() + repository + .audit_events() .iter() - .any(|action| action == "backup.delete_requested") + .any(|event| event.action == "backup.delete_requested" + && event.entity_ref == created.id + && event.changed_fields == ["status"]) ); } diff --git a/backend/crates/gateway-admin/tests/use_case/mod.rs b/backend/crates/gateway-admin/tests/use_case/mod.rs index 4ad69d805..bb45b5806 100644 --- a/backend/crates/gateway-admin/tests/use_case/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/mod.rs @@ -99,6 +99,7 @@ pub(super) struct AdminHarness { proxy_probe: Arc, account_runtime: Arc, account_groups: Arc, + snapshot: Arc, auth: Arc, client_keys: Arc, observability: Arc, @@ -125,6 +126,7 @@ impl AdminHarness { proxy_probe: Arc::new(proxies::TestProxies::default()), account_runtime: unavailable.clone(), account_groups: Arc::new(UnavailableAccountGroupStore), + snapshot: Arc::new(NoopSnapshot), auth: Arc::new(BootstrapAuthStore::default()), client_keys: unavailable.clone(), observability: unavailable.clone(), @@ -168,6 +170,11 @@ impl AdminHarness { self } + pub(super) fn snapshot(mut self, snapshot: Arc) -> Self { + self.snapshot = snapshot; + self + } + pub(super) fn auth(mut self, store: Arc) -> Self { self.auth = store; self @@ -291,7 +298,7 @@ impl AdminHarness { self.providers, ) .unwrap(), - snapshot: Arc::new(NoopSnapshot), + snapshot: self.snapshot, account_probe: self.probe, proxy_probe: self.proxy_probe, client_distribution: Arc::new(NoopClientDistribution), diff --git a/backend/crates/gateway-api/src/admin/plugins/management.rs b/backend/crates/gateway-api/src/admin/plugins/management.rs index 405a30749..878334fe9 100644 --- a/backend/crates/gateway-api/src/admin/plugins/management.rs +++ b/backend/crates/gateway-api/src/admin/plugins/management.rs @@ -44,6 +44,8 @@ use crate::{ const MAXIMUM_BODY_BYTES: usize = 1024 * 1024; const MAXIMUM_MODEL_BODY_BYTES: usize = 8 * 1024 * 1024; +// 传输归属由宿主写入，不能用插件可覆盖的安全响应头识别管理错误 +const PLUGIN_RESPONSE_HEADER: &str = "x-cpr-plugin-response"; // HTML/SVG 即使被直接导航打开也不能获得管理端 origin；执行入口由隔离 iframe 的消息桥承载 const RESOURCE_CSP: &str = "sandbox allow-scripts; default-src 'none'; script-src 'unsafe-inline' blob:; style-src 'unsafe-inline' blob:; img-src data: blob:; font-src data: blob:; connect-src 'none'; frame-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'self'"; @@ -124,17 +126,17 @@ async fn model_responses( match decode_request_with_body(&body, &headers, MAXIMUM_MODEL_BODY_BYTES) { Ok(decoded) => decoded, Err(error) => { - return Ok(protocol_error_response( + return Ok(plugin_response(protocol_error_response( StatusCode::BAD_REQUEST, error.protocol_body(), - )); + ))); } }; let service = state.openai().clone(); let execution = service.execution(); let prepared = match execution.prepare_plugin_execution(&client_key_id).await { Ok(prepared) => prepared, - Err(error) => return Ok(gateway_error_response(&error)), + Err(error) => return Ok(plugin_response(gateway_error_response(&error))), }; let authorization: Arc = Arc::new(PluginModelsAuthorization { state, target }); @@ -158,7 +160,7 @@ async fn model_responses( Some(authorization), ) .await; - Ok(response) + Ok(plugin_response(response)) } struct PluginModelsAuthorization { @@ -431,5 +433,12 @@ fn raw_response(result: PluginManagementResponse) -> Result Response { + response + .headers_mut() + .insert(PLUGIN_RESPONSE_HEADER, HeaderValue::from_static("1")); + response } diff --git a/backend/crates/gateway-api/src/openai/responses/http.rs b/backend/crates/gateway-api/src/openai/responses/http.rs index 3bc0a5483..406f3d791 100644 --- a/backend/crates/gateway-api/src/openai/responses/http.rs +++ b/backend/crates/gateway-api/src/openai/responses/http.rs @@ -321,24 +321,36 @@ impl AuthorizedResponseBody { impl MiddlewareBody for AuthorizedResponseBody { fn next_frame(&mut self) -> BoxFuture<'_, Result, MiddlewareError>> { Box::pin(async move { - loop { - let elapsed = self.last_authorized_at.elapsed(); - if elapsed >= AUTHORIZATION_RECHECK_INTERVAL { - if let Err(error) = self.authorization.authorize().await { - return Err(self.revoke(error).await); - } - self.last_authorized_at = Instant::now(); - continue; + if self.last_authorized_at.elapsed() >= AUTHORIZATION_RECHECK_INTERVAL { + if let Err(error) = self.authorization.authorize().await { + return Err(self.revoke(error).await); } + self.last_authorized_at = Instant::now(); + } + let error = { let Some(inner) = self.inner.as_mut() else { return Ok(None); }; - let wait = AUTHORIZATION_RECHECK_INTERVAL - elapsed; - tokio::select! { - frame = inner.next_frame() => return frame, - () = tokio::time::sleep(wait) => {} + // 复核身份只暂停同一次读取，不能丢弃已经消费上游事件的 future + let frame = inner.next_frame(); + tokio::pin!(frame); + loop { + let elapsed = self.last_authorized_at.elapsed(); + if elapsed >= AUTHORIZATION_RECHECK_INTERVAL { + if let Err(error) = self.authorization.authorize().await { + break error; + } + self.last_authorized_at = Instant::now(); + continue; + } + let wait = AUTHORIZATION_RECHECK_INTERVAL - elapsed; + tokio::select! { + result = &mut frame => return result, + () = tokio::time::sleep(wait) => {} + } } - } + }; + Err(self.revoke(error).await) }) } diff --git a/backend/crates/gateway-api/tests/admin/plugins/management.rs b/backend/crates/gateway-api/tests/admin/plugins/management.rs index 6d08000a7..94b5e1cfd 100644 --- a/backend/crates/gateway-api/tests/admin/plugins/management.rs +++ b/backend/crates/gateway-api/tests/admin/plugins/management.rs @@ -32,7 +32,7 @@ use gateway_core::routing::extensions::ExtensionSetLease; use gateway_core::routing::extensions::ExtensionSetReference; use std::sync::{ Arc, Mutex, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, }; use tower::ServiceExt as _; @@ -41,6 +41,7 @@ use super::super::AdminTestFixture; #[derive(Debug, Clone, Copy)] enum ModelReply { Complete, + SessionError, PendingResponse, PendingBody, } @@ -50,6 +51,7 @@ struct ModelExecution { cancellation: Mutex>, entered: Arc, closed: Arc, + body_control: Arc, } impl ModelExecution { @@ -59,6 +61,7 @@ impl ModelExecution { cancellation: Mutex::new(None), entered: Arc::default(), closed: Arc::default(), + body_control: Arc::default(), }) } } @@ -92,6 +95,7 @@ impl ExecutionService for ModelExecution { reply: self.reply, entered: self.entered.clone(), closed: self.closed.clone(), + body_control: self.body_control.clone(), }), ExtensionSetReference::new( ExtensionSetId::new("model-bridge-test".into()).unwrap(), @@ -122,6 +126,7 @@ struct ModelMiddleware { reply: ModelReply, entered: Arc, closed: Arc, + body_control: Arc, } impl MiddlewarePlan for ModelMiddleware { @@ -152,12 +157,19 @@ impl MiddlewarePlan for ModelMiddleware { let reply = self.reply; let closed = self.closed.clone(); let entered = self.entered.clone(); + let body_control = self.body_control.clone(); Box::pin(async move { entered.notify_one(); if matches!(reply, ModelReply::PendingResponse) { return futures::future::pending().await; } - let (bytes, framing, terminal) = if matches!(reply, ModelReply::PendingBody) { + let (bytes, framing, terminal) = if matches!(reply, ModelReply::SessionError) { + ( + "{\"code\":40101,\"message\":\"plugin error\",\"data\":null}", + MiddlewareFraming::JsonDocument, + true, + ) + } else if matches!(reply, ModelReply::PendingBody) { ( "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_bridge\",\"object\":\"response\",\"status\":\"in_progress\",\"output\":[]}}\n\n", MiddlewareFraming::SseEvent, @@ -172,32 +184,67 @@ impl MiddlewarePlan for ModelMiddleware { }; Ok(MiddlewareResponse::new( "openai".into(), - 200, + if matches!(reply, ModelReply::SessionError) { + 401 + } else { + 200 + }, vec![ MiddlewareHeader::new("set-cookie", "fixture=value".into()), MiddlewareHeader::new("www-authenticate", "Bearer".into()), MiddlewareHeader::new("x-plugin-response", "complete".into()), + MiddlewareHeader::new("x-cpr-plugin-response", "untrusted".into()), ], Box::new(ModelBody { frame: Some(MiddlewareFrame::new(bytes.into(), framing, terminal)), pending: !terminal, closed, + control: body_control, }), )) }) } } +#[derive(Debug, Default)] +struct ModelBodyControl { + delayed_frame: AtomicBool, + pending_reads: AtomicUsize, + waiting: tokio::sync::Notify, + release: tokio::sync::Notify, +} + struct ModelBody { frame: Option, pending: bool, closed: Arc, + control: Arc, } impl MiddlewareBody for ModelBody { fn next_frame(&mut self) -> BoxFuture<'_, Result, MiddlewareError>> { + let frame = self.frame.take(); + let delayed_frame = if frame.is_none() && self.pending { + self.control.pending_reads.fetch_add(1, Ordering::SeqCst); + self.control.waiting.notify_one(); + // 模拟读取在 future 创建时已经消费事件，取消后不能重新取得原帧 + self.control.delayed_frame.swap(false, Ordering::SeqCst).then(|| { + MiddlewareFrame::new( + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_bridge\",\"object\":\"response\",\"status\":\"completed\",\"output\":[]}}\n\n".into(), + MiddlewareFraming::SseEvent, + true, + ) + }) + } else { + None + }; Box::pin(async move { - if let Some(frame) = self.frame.take() { + if let Some(frame) = frame { + return Ok(Some(frame)); + } + if let Some(frame) = delayed_frame { + self.control.release.notified().await; + self.pending = false; return Ok(Some(frame)); } if self.pending { @@ -245,6 +292,7 @@ async fn management_calls_and_callbacks_preserve_headers_and_override_defaults() .header("set-cookie", "first=1") .header("set-cookie", "second=2") .header("content-security-policy", "default-src 'self'") + .header("x-cpr-plugin-response", "untrusted") .body(Body::empty()) .unwrap(), ) @@ -273,6 +321,15 @@ async fn management_calls_and_callbacks_preserve_headers_and_override_defaults() response.headers()["content-security-policy"], "default-src 'self'" ); + assert_eq!(response.headers()["x-cpr-plugin-response"], "1"); + assert_eq!( + response + .headers() + .get_all("x-cpr-plugin-response") + .iter() + .count(), + 1 + ); } } @@ -288,6 +345,7 @@ async fn model_bridge_preserves_complete_request_and_response_headers() { assert_eq!(response.headers()["set-cookie"], "fixture=value"); assert_eq!(response.headers()["www-authenticate"], "Bearer"); assert_eq!(response.headers()["x-plugin-response"], "complete"); + assert_eq!(response.headers()["x-cpr-plugin-response"], "1"); assert_eq!(response.headers()["cache-control"], "no-store"); let body = axum::body::to_bytes(response.into_body(), 4096) .await @@ -298,6 +356,40 @@ async fn model_bridge_preserves_complete_request_and_response_headers() { ); } +#[tokio::test] +async fn model_bridge_distinguishes_host_authentication_from_plugin_session_errors() { + let fixture = published_fixture().await; + let router = crate::openai::api_router_with_admin_and_execution( + fixture.services, + ModelExecution::new(ModelReply::SessionError), + ); + let mut unauthenticated = model_request(false); + unauthenticated.headers_mut().remove("cookie"); + unauthenticated.headers_mut().remove("authorization"); + unauthenticated.headers_mut().remove("x-api-key"); + let response = router.clone().oneshot(unauthenticated).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(!response.headers().contains_key("x-cpr-plugin-response")); + let body = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["code"], + 40101 + ); + + let response = router.oneshot(model_request(false)).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.headers()["x-cpr-plugin-response"], "1"); + let body = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["code"], + 40101 + ); +} + #[tokio::test(start_paused = true)] async fn model_bridge_revokes_while_waiting_for_the_first_response() { let fixture = published_fixture().await; @@ -358,6 +450,52 @@ async fn model_bridge_revokes_and_closes_a_pending_stream() { assert!(execution.closed.load(Ordering::SeqCst)); } +#[tokio::test(start_paused = true)] +async fn model_bridge_authorization_rechecks_preserve_the_pending_frame_read() { + let fixture = published_fixture().await; + let execution = ModelExecution::new(ModelReply::PendingBody); + execution + .body_control + .delayed_frame + .store(true, Ordering::SeqCst); + let response = + crate::openai::api_router_with_admin_and_execution(fixture.services, execution.clone()) + .oneshot(model_request(true)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = tokio::spawn(axum::body::to_bytes(response.into_body(), 4096)); + tokio::time::timeout( + std::time::Duration::from_secs(5), + execution.body_control.waiting.notified(), + ) + .await + .expect("第二帧读取已开始"); + for _ in 0..3 { + tokio::time::advance(std::time::Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert_eq!( + execution.body_control.pending_reads.load(Ordering::SeqCst), + 1 + ); + assert!(!body.is_finished()); + } + execution.body_control.release.notify_one(); + let body = tokio::time::timeout(std::time::Duration::from_secs(5), body) + .await + .expect("原先等待的终态帧应完成交付") + .unwrap() + .unwrap(); + let body = std::str::from_utf8(&body).unwrap(); + assert_eq!(body.matches("event: response.completed").count(), 1); + assert!(!body.contains("response.failed")); + assert_eq!( + execution.body_control.pending_reads.load(Ordering::SeqCst), + 1 + ); + assert!(execution.closed.load(Ordering::SeqCst)); +} + pub(super) fn resource_fixture( path: &str, ) -> gateway_admin::model::plugins::management::PluginManagementResponse { diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index de5ed931b..554f6edff 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -385,6 +385,7 @@ where last_retryable_failure_events: Vec::new(), last_observed_provider: None, pending_terminal_failure: None, + pending_stream_failure: None, client_retry: None, }; @@ -555,12 +556,38 @@ pub struct ResponseExecutionSession { last_observed_provider: Option, /// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误 pending_terminal_failure: Option, + /// 已消费的流错误及其异步决策由会话持有，取消读取只暂停处理 + pending_stream_failure: Option, client_retry: Option, } struct PendingTerminalFailure { - error: ProviderError, + error: Arc, send_state: UpstreamSendState, + events: Vec, +} + +struct PendingStreamFailure { + completion: StreamFailureDecision, + error: ProviderError, + events: Vec, + send_state: UpstreamSendState, + retry: StreamRetry, + trace: serde_json::Value, +} + +enum StreamFailureDecision { + Pending(BoxFuture<'static, Result>), + Retry(bool), + Interrupted(Arc), +} + +enum StreamRetry { + SameAccount, + Pending(PendingAttemptRetry), + ConnectionRotation(PendingAttemptRetry), + ExcludeAccount, + Continuation, } enum RequestFinalization { @@ -602,14 +629,14 @@ where if self.delivery_pending { return Err(EngineError::DownstreamCommitRequired); } - if let Some(pending) = self.pending_terminal_failure.take() { - self.finish_provider_error_with_send_state(&pending.error, pending.send_state) - .await?; - return Err(provider_engine_error(pending.error)); - } if self.is_finalized() { + self.pending_terminal_failure.take(); + self.pending_stream_failure.take(); return Ok(None); } + if self.pending_terminal_failure.is_some() { + return self.next_terminal_failure().await; + } loop { match self.pull().await? { PullOutcome::Events(events) => { @@ -629,17 +656,12 @@ where error, send_state, } => { - self.record_client_retry_failure(&error).await; - let requirement = if self.delivery_committed { - CommitRequirement::AlreadyCommitted - } else { - self.delivery_pending = true; - CommitRequirement::CommitBeforeDelivery - }; - self.replay_barrier_reached = true; - self.pending_terminal_failure = - Some(PendingTerminalFailure { error, send_state }); - return CoordinatedEvent::try_batch(events, requirement).map(Some); + self.pending_terminal_failure = Some(PendingTerminalFailure { + error: Arc::new(error), + send_state, + events, + }); + return self.next_terminal_failure().await; } PullOutcome::End => { if self.delivery_committed { @@ -673,9 +695,14 @@ where PullOutcome::TerminalFailure { error, send_state, .. } => { - self.finish_provider_error_with_send_state(&error, send_state) - .await?; - return Err(provider_engine_error(error)); + self.pending_terminal_failure = Some(PendingTerminalFailure { + error: Arc::new(error), + send_state, + events: Vec::new(), + }); + return Err(provider_engine_error( + self.finalize_pending_terminal_failure().await?, + )); } PullOutcome::End => { if events.is_empty() { @@ -873,21 +900,25 @@ where /// 观测写入失败只记录告警，不改变取消结果 pub async fn cancel_and_finalize(&mut self) -> Result<(), EngineError> { self.cancellation.cancel(); + // 已观测事实归属 session，显式取消不再保留失败决策中的存储或插件调用 + self.pending_stream_failure.take(); self.resume_finalization().await; if self.is_finalized() { return Ok(()); } self.delivery_pending = false; - if let Some(pending) = self.pending_terminal_failure.take() { - return self - .finish_provider_error_with_send_state(&pending.error, pending.send_state) - .await; + if self.pending_terminal_failure.is_some() { + self.finalize_pending_terminal_failure().await?; + return Ok(()); } self.finish_interruption(&EngineError::Cancelled).await } async fn pull(&mut self) -> Result { loop { + if self.pending_stream_failure.is_some() { + return self.resume_stream_failure().await; + } if self.current.is_none() { match self.prepare_attempt().await { Ok(Some(outcome)) => return Ok(outcome), @@ -919,7 +950,7 @@ where } PollBoundary::Deadline => { // 会话 deadline 是网关自身的请求预算，不是上游超时； - // 真正的上游超时会作为流错误进入 `handle_stream_error` 记账 + // 真正的上游超时会作为流错误进入 `begin_stream_failure` 记账 // 这里不写 provider 失败，避免把本地预算到期归因为上游故障 self.finish_interruption(&EngineError::Deadline).await?; return Err(EngineError::Deadline); @@ -981,14 +1012,8 @@ where } } PollBoundary::Item(Some(Err(error))) => { - match self.handle_stream_error(error).await? { - StreamErrorOutcome::AttemptDiscarded => { - return Ok(PullOutcome::AttemptDiscarded); - } - terminal @ StreamErrorOutcome::TerminalFailure { .. } => { - return Ok(terminal.into_pull_outcome()); - } - } + self.begin_stream_failure(error)?; + return self.resume_stream_failure().await; } PollBoundary::Item(None) => { self.record_current_provider_success(); @@ -1572,21 +1597,23 @@ where } /// 失败可重试时要求丢弃本 attempt；预算耗尽时可返回最后一批原始失败事件 - async fn handle_stream_error( - &mut self, - mut error: ProviderError, - ) -> Result { + fn begin_stream_failure(&mut self, mut error: ProviderError) -> Result<(), EngineError> { // 原始 wire 只活在 request-local 决策状态；clone、attempt 记录与持久化终态 // 均只接触已剥离的稳定错误字段 self.trace .attempt(self.attempts) .record_provider_failure(&error); - let mut atomic_client_events = error.take_atomic_client_events(); - // 失败随附的观测也属于当前 attempt，换号记录不能等到最终交付才读取 + let atomic_client_events = error.take_atomic_client_events(); + // 失败随附的事实也属于当前 attempt，取消观测写入时仍须保留 usage 与费用 + // 回退仍可能交付该批事件，因此这里只借用观测，确认终态交付后再移除 for event in &atomic_client_events { if let Some(observation) = event.response_observation() { self.observe_response(observation.clone()); } + for fact in event.canonical_facts() { + self.observation.observe_identity(fact); + self.observation.observe_event(fact); + } } let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?; if self.request_observation.is_some() { @@ -1614,16 +1641,6 @@ where } let execution_effect_observed = self.execution_effect_observed(); let send_state = self.raise_send_watermark(attempt_send_state); - if self.request_persisted { - best_effort_store_write( - "mark_send_state", - &self.request_id, - self.engine - .store() - .mark_send_state(&self.request_id, send_state), - ) - .await; - } let provider_proved_replay_safe = provider_proved_replay_safe(&error); let continuation_retry = !execution_effect_observed && self.prepare_continuation_retry( @@ -1778,36 +1795,25 @@ where || connection_rotation.is_some() || rotation_retry); - let retryable = match self - .apply_retry_policy(super::policy::RetryFacts { - attempt_index: current.index, - provider: current.metadata.provider().clone(), - model: current - .metadata - .upstream_model() - .map(|model| model.as_str().to_owned()), - error_kind: error.kind(), - upstream_status: error.upstream_status(), - send_state: attempt_send_state, - remaining_routing_attempts: self - .plan - .max_attempts() - .get() - .saturating_sub(self.routing_attempts), - remaining_deadline: Duration::ZERO, - retry_allowed: retryable, - }) - .await - { - Ok(retryable) => retryable, - Err(error) => { - self.current = Some(current); - self.finish_interruption(&error).await?; - return Err(error); - } - }; - - self.trace.attempt(current.index.get()).record("retry.decided", json!({ + let policy = self.apply_retry_policy(super::policy::RetryFacts { + attempt_index: current.index, + provider: current.metadata.provider().clone(), + model: current + .metadata + .upstream_model() + .map(|model| model.as_str().to_owned()), + error_kind: error.kind(), + upstream_status: error.upstream_status(), + send_state: attempt_send_state, + remaining_routing_attempts: self + .plan + .max_attempts() + .get() + .saturating_sub(self.routing_attempts), + remaining_deadline: Duration::ZERO, + retry_allowed: retryable, + }); + let trace = json!({ "phase": "execute", "retryable": retryable, "continuationRetry": continuation_retry, "sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry, @@ -1836,145 +1842,269 @@ where "credentialRecoveryAttempted": self.credential_recovery_attempted_accounts.contains(current.metadata.provider_account_id()), "hasRetryPolicy": self.request_policy.is_some(), "sendState": format!("{attempt_send_state:?}"), - })); - if retryable { - for event in &atomic_client_events { - for fact in event.canonical_facts() { - self.observation.observe_event(fact); - } - } - // 原始 wire/HTTP response 由 request-local 所有权保留到下一次 attempt - // 成功，或最终空选路时返回客户端；持久化只取得稳定事实快照 - let persistence_error = self.request_persisted.then(|| error.stable_snapshot()); - if same_account_retry { - let account = current.metadata.provider_account_id().clone(); - self.credential_recovery_attempted_accounts - .insert(account.clone()); - // 只钉住紧随其后的 replay attempt；replay 再遇可重试错误时， - // ordinary/continuation 重试门不受影响，仍可换号 - self.recovery_account = Some(account); - } else if let Some(delay) = transient_retry { - self.pending_retry = Some(PendingAttemptRetry { - account: current.metadata.provider_account_id().clone(), - transport: current.transport, - delay, - transport_recovery: false, - reserve_connection_rotation: false, - }); - } else if let Some((transport, delay)) = transport_recovery { - self.pending_retry = Some(PendingAttemptRetry { - account: current.metadata.provider_account_id().clone(), - transport, - delay, - transport_recovery: true, - reserve_connection_rotation, - }); - } else if let Some((transport, delay)) = connection_rotation { - let account = current.metadata.provider_account_id().clone(); - self.excluded_accounts.insert(account.clone()); - self.connection_rotation = Some(PendingAttemptRetry { - account, - transport, - delay, - transport_recovery: true, - reserve_connection_rotation: false, - }); - } else if !continuation_retry { - self.excluded_accounts - .insert(current.metadata.provider_account_id().clone()); - } - self.last_retryable_failure_events = atomic_client_events; - self.last_retryable_failure = Some(error); - if let Some(error) = persistence_error { - best_effort_store_write( - "record_intermediate_failure", - &self.request_id, - self.engine - .store() - .record_intermediate_failure(IntermediateFailure { - account_snapshot: current.metadata.account_snapshot().cloned(), - request_id: self.request_id.clone(), - attempt_index: current.index, - trigger: current.trigger, - provider_kind: current.metadata.provider().clone(), - account_id: Some(current.metadata.provider_account_id().clone()), - upstream_model_id: current.metadata.upstream_model().cloned(), - upstream_status_code: error.upstream_status().or_else(|| { - current - .response_observation - .as_ref() - .and_then(ProviderResponseObservation::status_code) - }), - upstream_request_id: error - .upstream_request_id() - .map(|id| id.as_str()) - .or_else(|| current.upstream_request_id()) - .map(str::to_owned), - latency: current.started_at.elapsed().unwrap_or_default(), - error, - }), - ) - .await; - } - self.reset_uncommitted_observations(); - return Ok(StreamErrorOutcome::AttemptDiscarded); - } - - self.current = Some(current); - if !atomic_client_events.is_empty() { - self.observe_atomic_terminal_events(&mut atomic_client_events); - return Ok(StreamErrorOutcome::TerminalFailure { - events: atomic_client_events, + }); + let retry = if same_account_retry { + StreamRetry::SameAccount + } else if let Some(delay) = transient_retry { + StreamRetry::Pending(PendingAttemptRetry { + account: current.metadata.provider_account_id().clone(), + transport: current.transport, + delay, + transport_recovery: false, + reserve_connection_rotation: false, + }) + } else if let Some((transport, delay)) = transport_recovery { + StreamRetry::Pending(PendingAttemptRetry { + account: current.metadata.provider_account_id().clone(), + transport, + delay, + transport_recovery: true, + reserve_connection_rotation, + }) + } else if let Some((transport, delay)) = connection_rotation { + StreamRetry::ConnectionRotation(PendingAttemptRetry { + account: current.metadata.provider_account_id().clone(), + transport, + delay, + transport_recovery: true, + reserve_connection_rotation: false, + }) + } else if continuation_retry { + StreamRetry::Continuation + } else { + StreamRetry::ExcludeAccount + }; + let intermediate = self.request_persisted.then(|| { + let error = error.stable_snapshot(); + IntermediateFailure { + account_snapshot: current.metadata.account_snapshot().cloned(), + request_id: self.request_id.clone(), + attempt_index: current.index, + trigger: current.trigger, + provider_kind: current.metadata.provider().clone(), + account_id: Some(current.metadata.provider_account_id().clone()), + upstream_model_id: current.metadata.upstream_model().cloned(), + upstream_status_code: error.upstream_status().or_else(|| { + current + .response_observation + .as_ref() + .and_then(ProviderResponseObservation::status_code) + }), + upstream_request_id: error + .upstream_request_id() + .map(|id| id.as_str()) + .or_else(|| current.upstream_request_id()) + .map(str::to_owned), + latency: current.started_at.elapsed().unwrap_or_default(), error, - send_state, - }); - } - self.finish_provider_error_with_send_state(&error, send_state) - .await?; - Err(provider_engine_error(error)) + } + }); + let store = Arc::clone(self.engine.store()); + let request_id = self.request_id.clone(); + let persisted = self.request_persisted; + // 错误、账号 lease 与决策 future 都跨 next_event 的取消保留 + // 在持久化及策略回调完成前，不能再次拉流或建立新的 attempt + self.current = Some(current); + self.pending_stream_failure = Some(PendingStreamFailure { + completion: StreamFailureDecision::Pending(Box::pin(async move { + if persisted { + best_effort_store_write( + "mark_send_state", + &request_id, + store.mark_send_state(&request_id, send_state), + ) + .await; + } + let retryable = policy.await?; + if retryable && let Some(intermediate) = intermediate { + best_effort_store_write( + "record_intermediate_failure", + &request_id, + store.record_intermediate_failure(intermediate), + ) + .await; + } + Ok(retryable) + })), + error, + events: atomic_client_events, + send_state, + retry, + trace, + }); + Ok(()) } - async fn apply_retry_policy( - &mut self, - mut facts: super::policy::RetryFacts, - ) -> Result { - let Some(policy) = self.request_policy.clone() else { - return Ok(facts.retry_allowed); + async fn resume_stream_failure(&mut self) -> Result { + let pending = self + .pending_stream_failure + .as_mut() + .ok_or(EngineError::NoActiveAttempt)?; + if let StreamFailureDecision::Pending(completion) = &mut pending.completion { + pending.completion = match completion.as_mut().await { + Ok(retryable) => StreamFailureDecision::Retry(retryable), + Err(error) => StreamFailureDecision::Interrupted(Arc::new(error)), + }; + } + if let StreamFailureDecision::Interrupted(error) = &pending.completion { + let error = Arc::clone(error); + self.finish_interruption(&error).await?; + self.pending_stream_failure.take(); + return Err(Arc::try_unwrap(error).expect("interruption is owned by the session")); + } + let mut pending = self + .pending_stream_failure + .take() + .ok_or(EngineError::NoActiveAttempt)?; + let StreamFailureDecision::Retry(retryable) = pending.completion else { + unreachable!("completed stream failure has a retry decision"); }; - facts.remaining_deadline = self.deadline.remaining().unwrap_or(Duration::MAX); - facts.retry_allowed &= !facts.remaining_deadline.is_zero() - && !self.cancellation.is_cancelled() - && !self.execution_effect_observed(); - let allowed = facts.retry_allowed; - let attempt_index = facts.attempt_index; - let decision = { - let cancellation = self.cancellation.clone(); - let cancelled = cancellation.cancelled().fuse(); - let decision = policy.retry_decision(facts).fuse(); - let mut deadline_timer = &mut self.deadline_timer; - pin_mut!(cancelled, decision); - select_biased! { - () = cancelled => Err(EngineError::Cancelled), - () = deadline_timer => Err(EngineError::Deadline), - decision = decision => Ok(decision), + pending.trace["retryable"] = json!(retryable); + self.trace + .attempt(self.attempts) + .record("retry.decided", pending.trace); + if retryable { + let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?; + match pending.retry { + StreamRetry::SameAccount => { + let account = current.metadata.provider_account_id().clone(); + self.credential_recovery_attempted_accounts + .insert(account.clone()); + self.recovery_account = Some(account); + } + StreamRetry::Pending(retry) => self.pending_retry = Some(retry), + StreamRetry::ConnectionRotation(retry) => { + self.excluded_accounts.insert(retry.account.clone()); + self.connection_rotation = Some(retry); + } + StreamRetry::ExcludeAccount => { + self.excluded_accounts + .insert(current.metadata.provider_account_id().clone()); + } + StreamRetry::Continuation => {} } - }?; - self.trace.attempt(attempt_index.get()).record( - "retry.policy", - json!({ - "hostAllowed": allowed, - "policyStopped": decision == super::policy::RetryDecision::Stop, - "executionEffectObserved": self.execution_effect_observed(), - "cancelled": self.cancellation.is_cancelled(), - "deadlineElapsed": self.deadline.is_elapsed(), - }), - ); - // 策略只收窄宿主已允许的恢复，返回后仍复核期限和外部副作用 - Ok(allowed - && decision != super::policy::RetryDecision::Stop - && !self.execution_effect_observed() - && !self.cancellation.is_cancelled() - && !self.deadline.is_elapsed()) + self.last_retryable_failure_events = pending.events; + self.last_retryable_failure = Some(pending.error); + self.reset_uncommitted_observations(); + return Ok(PullOutcome::AttemptDiscarded); + } + if !pending.events.is_empty() { + for event in &mut pending.events { + let _ = event.take_observation(); + } + return Ok(PullOutcome::TerminalFailure { + events: pending.events, + error: pending.error, + send_state: pending.send_state, + }); + } + self.pending_terminal_failure = Some(PendingTerminalFailure { + error: Arc::new(pending.error), + send_state: pending.send_state, + events: Vec::new(), + }); + Err(provider_engine_error( + self.finalize_pending_terminal_failure().await?, + )) + } + + async fn next_terminal_failure(&mut self) -> Result, EngineError> { + let pending = self + .pending_terminal_failure + .as_ref() + .ok_or(EngineError::NoActiveAttempt)?; + if pending.events.is_empty() { + return Err(provider_engine_error( + self.finalize_pending_terminal_failure().await?, + )); + } + let error = Arc::clone(&pending.error); + self.record_client_retry_failure(&error).await; + let pending = self + .pending_terminal_failure + .as_mut() + .ok_or(EngineError::NoActiveAttempt)?; + let events = std::mem::take(&mut pending.events); + let requirement = if self.delivery_committed { + CommitRequirement::AlreadyCommitted + } else { + self.delivery_pending = true; + CommitRequirement::CommitBeforeDelivery + }; + self.replay_barrier_reached = true; + CoordinatedEvent::try_batch(events, requirement).map(Some) + } + + async fn finalize_pending_terminal_failure(&mut self) -> Result { + let pending = self + .pending_terminal_failure + .as_ref() + .ok_or(EngineError::NoActiveAttempt)?; + let error = Arc::clone(&pending.error); + let send_state = pending.send_state; + self.finish_provider_error_with_send_state(&error, send_state) + .await?; + drop(error); + let pending = self + .pending_terminal_failure + .take() + .ok_or(EngineError::NoActiveAttempt)?; + Ok(Arc::try_unwrap(pending.error).expect("terminal failure is owned by the session")) + } + + fn apply_retry_policy( + &self, + mut facts: super::policy::RetryFacts, + ) -> BoxFuture<'static, Result> { + let policy = self.request_policy.clone(); + let deadline = self.deadline; + let cancellation = self.cancellation.clone(); + let effects = self.execution_effects.clone(); + let baseline = self.execution_effects_baseline; + let trace = self.trace.clone(); + Box::pin(async move { + let Some(policy) = policy else { + return Ok(facts.retry_allowed); + }; + let effect_observed = || { + effects + .as_ref() + .is_some_and(|effects| effects.epoch() != baseline) + }; + facts.remaining_deadline = deadline.remaining().unwrap_or(Duration::MAX); + facts.retry_allowed &= !facts.remaining_deadline.is_zero() + && !cancellation.is_cancelled() + && !effect_observed(); + let allowed = facts.retry_allowed; + let attempt_index = facts.attempt_index; + let decision = { + let cancelled = cancellation.cancelled().fuse(); + let decision = policy.retry_decision(facts).fuse(); + let mut deadline_timer = deadline.wait().fuse(); + pin_mut!(cancelled, decision); + select_biased! { + () = cancelled => Err(EngineError::Cancelled), + () = deadline_timer => Err(EngineError::Deadline), + decision = decision => Ok(decision), + } + }?; + trace.attempt(attempt_index.get()).record( + "retry.policy", + json!({ + "hostAllowed": allowed, + "policyStopped": decision == super::policy::RetryDecision::Stop, + "executionEffectObserved": effect_observed(), + "cancelled": cancellation.is_cancelled(), + "deadlineElapsed": deadline.is_elapsed(), + }), + ); + // 策略只收窄宿主已允许的恢复，返回后仍复核期限和外部副作用 + Ok(allowed + && decision != super::policy::RetryDecision::Stop + && !effect_observed() + && !cancellation.is_cancelled() + && !deadline.is_elapsed()) + }) } fn observe_atomic_terminal_events(&mut self, events: &mut [ProviderEvent]) { @@ -2604,32 +2734,6 @@ enum PullOutcome { End, } -enum StreamErrorOutcome { - AttemptDiscarded, - TerminalFailure { - events: Vec, - error: ProviderError, - send_state: UpstreamSendState, - }, -} - -impl StreamErrorOutcome { - fn into_pull_outcome(self) -> PullOutcome { - match self { - Self::AttemptDiscarded => PullOutcome::AttemptDiscarded, - Self::TerminalFailure { - events, - error, - send_state, - } => PullOutcome::TerminalFailure { - events, - error, - send_state, - }, - } - } -} - enum PollBoundary { Item(Option>), Cancelled, diff --git a/backend/crates/gateway-core/src/engine/provider.rs b/backend/crates/gateway-core/src/engine/provider.rs index 2b59ed58d..f3dcf084a 100644 --- a/backend/crates/gateway-core/src/engine/provider.rs +++ b/backend/crates/gateway-core/src/engine/provider.rs @@ -3,13 +3,13 @@ use std::collections::{BTreeMap, VecDeque}; use std::fmt; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use std::time::Instant; use async_trait::async_trait; use bytes::Bytes; -use futures::{Stream, StreamExt, future::BoxFuture}; +use futures::{Stream, StreamExt, TryStreamExt, channel::oneshot, future::BoxFuture}; use thiserror::Error; use crate::account::{ @@ -485,6 +485,9 @@ fn provider_middleware_next( operation: Operation, transport: ClientTransport, terminal: ProviderMiddlewareTerminal, + prepared: oneshot::Sender, + activate: oneshot::Receiver<()>, + observation: Arc>, ) -> MiddlewareNext { crate::middleware::compose(Vec::new(), move |request| { Box::pin(async move { @@ -501,11 +504,17 @@ fn provider_middleware_next( .await .map_err(MiddlewareError::Provider)?; let metadata = stream.metadata().clone(); + prepared + .send(metadata.clone()) + .map_err(|_| MiddlewareError::InvalidState)?; + // Core 先校验 metadata 并登记 attempt，首次轮询外层冷流后才放行 next + activate.await.map_err(|_| MiddlewareError::InvalidState)?; let protocol = stream.middleware_protocol(&fallback_protocol); let body = ProviderMiddlewareBody { stream, pending: VecDeque::new(), transport, + observation, }; Ok( MiddlewareResponse::new(protocol, 200, Vec::new(), Box::new(body)) @@ -550,6 +559,40 @@ struct ProviderMiddlewareBody { stream: ProviderStream, pending: VecDeque<(ProviderEvent, bool)>, transport: ClientTransport, + observation: Arc>, +} + +struct MiddlewareSendObservation { + observed: UpstreamSendState, + reading: bool, +} + +impl MiddlewareSendObservation { + fn state(&self) -> UpstreamSendState { + if self.reading && self.observed == UpstreamSendState::NotSent { + UpstreamSendState::Ambiguous + } else { + self.observed + } + } + + fn complete(&mut self, observed: UpstreamSendState) { + self.reading = false; + self.observed = match (self.observed, observed) { + (UpstreamSendState::Sent, _) | (_, UpstreamSendState::Sent) => UpstreamSendState::Sent, + (UpstreamSendState::Ambiguous, _) | (_, UpstreamSendState::Ambiguous) => { + UpstreamSendState::Ambiguous + } + _ => UpstreamSendState::NotSent, + }; + } +} + +fn middleware_send_state(observation: &Mutex) -> UpstreamSendState { + observation + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .state() } impl MiddlewareBody for ProviderMiddlewareBody { @@ -561,7 +604,21 @@ impl MiddlewareBody for ProviderMiddlewareBody { .map(Some) .map_err(MiddlewareError::Provider); } - let Some(event) = self.stream.next().await else { + self.observation + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reading = true; + let event = self.stream.next().await; + let observed = match &event { + Some(Ok(_)) => UpstreamSendState::Sent, + Some(Err(error)) => error.send_state(), + None => UpstreamSendState::Ambiguous, + }; + self.observation + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .complete(observed); + let Some(event) = event else { return Ok(None); }; let event = event.map_err(MiddlewareError::Provider)?; @@ -689,30 +746,71 @@ pub async fn execute_attempt_middleware( .middleware_body() .map_err(|_| middleware_protocol_error(UpstreamSendState::NotSent))?, ); - let response = plan - .handle( - context, - request, - provider_middleware_next(operation, transport, terminal), - ) - .await - .map_err(middleware_prepare_error)?; - let (protocol, status_code, headers, body, envelope) = response.into_parts(); - if status_code != 200 || !headers.is_empty() { - return Err(middleware_protocol_error(UpstreamSendState::NotSent)); - } - crate::validation::validate_text(&protocol, 64, true, None) - .map_err(|_| middleware_protocol_error(UpstreamSendState::NotSent))?; - let metadata = envelope - .ok_or_else(|| middleware_protocol_error(UpstreamSendState::NotSent))? - .into_provider_metadata(); - let events = futures::stream::try_unfold((body, protocol), |(mut body, protocol)| async move { - let Some(frame) = body.next_frame().await.map_err(middleware_body_error)? else { - return Ok(None); - }; - let event = middleware_frame_to_provider_event(frame, &protocol)?; - Ok(Some((event, (body, protocol)))) - }); + let (prepared, metadata) = oneshot::channel(); + let (activate, activation) = oneshot::channel(); + let observation = Arc::new(Mutex::new(MiddlewareSendObservation { + observed: UpstreamSendState::NotSent, + reading: false, + })); + let response = plan.handle( + context, + request, + provider_middleware_next( + operation, + transport, + terminal, + prepared, + activation, + observation.clone(), + ), + ); + let (metadata, response) = match futures::future::select(metadata, response).await { + futures::future::Either::Left((Ok(metadata), response)) => (metadata, response), + futures::future::Either::Left((Err(_), response)) => { + return Err(unprepared_middleware_response(response.await)); + } + futures::future::Either::Right((response, _)) => { + return Err(unprepared_middleware_response(response)); + } + }; + let expected_metadata = metadata.clone(); + // 不创建后台任务，未激活的 handler、stream 与账号 lease 随返回的冷流一起回收 + let events = futures::stream::once(async move { + activate + .send(()) + .map_err(|_| middleware_protocol_error(middleware_send_state(&observation)))?; + let response = response + .await + .map_err(|error| middleware_error(error, middleware_send_state(&observation)))?; + let (protocol, status_code, headers, body, envelope) = response.into_parts(); + if status_code != 200 + || !headers.is_empty() + || envelope + .map(|envelope| envelope.into_provider_metadata()) + .as_ref() + != Some(&expected_metadata) + { + return Err(middleware_protocol_error(middleware_send_state( + &observation, + ))); + } + crate::validation::validate_text(&protocol, 64, true, None) + .map_err(|_| middleware_protocol_error(middleware_send_state(&observation)))?; + Ok(futures::stream::try_unfold( + (body, protocol, observation), + |(mut body, protocol, observation)| async move { + let Some(frame) = body.next_frame().await.map_err(|error| { + middleware_error(error, middleware_send_state(&observation)) + })? + else { + return Ok(None); + }; + let event = middleware_frame_to_provider_event(frame, &protocol)?; + Ok(Some((event, (body, protocol, observation)))) + }, + )) + }) + .try_flatten(); Ok(ProviderStream::new(metadata, events, ())) } @@ -786,37 +884,26 @@ fn middleware_frame_to_provider_event( Ok(event) } -fn middleware_prepare_error(error: MiddlewareError) -> ProviderError { - match error { - MiddlewareError::Provider(error) => error, - MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => { - ProviderError::new( - ProviderErrorKind::RequestPolicyDenied, - UpstreamSendState::NotSent, - ) - } - MiddlewareError::Fault - | MiddlewareError::Remote { .. } - | MiddlewareError::InvalidState - | MiddlewareError::Gateway(_) - | MiddlewareError::Engine(_) => middleware_protocol_error(UpstreamSendState::NotSent), +fn unprepared_middleware_response( + response: Result, +) -> ProviderError { + match response { + Err(error) => middleware_error(error, UpstreamSendState::NotSent), + Ok(_) => middleware_protocol_error(UpstreamSendState::NotSent), } } -fn middleware_body_error(error: MiddlewareError) -> ProviderError { +fn middleware_error(error: MiddlewareError, send_state: UpstreamSendState) -> ProviderError { match error { MiddlewareError::Provider(error) => error, MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => { - ProviderError::new( - ProviderErrorKind::RequestPolicyDenied, - UpstreamSendState::Ambiguous, - ) + ProviderError::new(ProviderErrorKind::RequestPolicyDenied, send_state) } MiddlewareError::Fault | MiddlewareError::Remote { .. } | MiddlewareError::InvalidState | MiddlewareError::Gateway(_) - | MiddlewareError::Engine(_) => middleware_protocol_error(UpstreamSendState::Ambiguous), + | MiddlewareError::Engine(_) => middleware_protocol_error(send_state), } } diff --git a/backend/crates/gateway-core/tests/engine/coordinator.rs b/backend/crates/gateway-core/tests/engine/coordinator.rs index ebe052723..8335196c6 100644 --- a/backend/crates/gateway-core/tests/engine/coordinator.rs +++ b/backend/crates/gateway-core/tests/engine/coordinator.rs @@ -530,6 +530,8 @@ struct FakeStore { failures: BTreeSet, create_gate: Mutex>>, finalize_gate: Mutex>>, + send_gate: Mutex>>, + intermediate_gate: Mutex>>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -587,6 +589,10 @@ impl ExecutionStore for FakeStore { .expect("store lock") .send_states .push(state); + let gate = self.send_gate.lock().expect("send gate lock").take(); + if let Some(gate) = gate { + gate.await.expect("send state write released"); + } Ok(()) } @@ -620,14 +626,24 @@ impl ExecutionStore for FakeStore { &self, failure: IntermediateFailure, ) -> Result<(), StoreError> { - let mut state = self.state.lock().expect("store lock"); - state.intermediate_failures += 1; - state - .intermediate_status_codes - .push(failure.upstream_status_code); - state - .intermediate_request_ids - .push(failure.upstream_request_id); + { + let mut state = self.state.lock().expect("store lock"); + state.intermediate_failures += 1; + state + .intermediate_status_codes + .push(failure.upstream_status_code); + state + .intermediate_request_ids + .push(failure.upstream_request_id); + } + let gate = self + .intermediate_gate + .lock() + .expect("intermediate gate lock") + .take(); + if let Some(gate) = gate { + gate.await.expect("intermediate failure write released"); + } Ok(()) } @@ -5458,6 +5474,169 @@ fn interrupted_first_attempt_write_never_restarts_selection_or_creates_a_zero_at }); } +#[test] +fn interrupted_stream_failure_preserves_the_original_attempt_and_retry_decision() { + block_on(async { + for (retryable, intermediate) in [(false, false), (true, false), (true, true)] { + let (release, gate) = oneshot::channel(); + let operation = generate_operation(); + let mut items = Vec::new(); + if !retryable { + items.push(Ok(GatewayEvent::Started(ResponseMeta::new( + "original", "gpt-5", + )))); + } + items.push(Err(ProviderError::new( + ProviderErrorKind::Transport, + if retryable { + UpstreamSendState::NotSent + } else { + UpstreamSendState::Sent + }, + ))); + let (coordinator, store, provider) = coordinator_with_store( + vec![ + Script::Stream { + account_id: "acct_first", + items, + }, + Script::Stream { + account_id: "acct_second", + items: complete_stream(None), + }, + ], + Arc::new(FakeStore::default()), + ); + let mut session = coordinator + .start( + model_request(&operation, SystemTime::now() + Duration::from_secs(30)), + operation.clone(), + plan(&operation), + None, + None, + CancellationToken::new(), + ) + .await + .unwrap(); + if !retryable { + session.next_event().await.unwrap().unwrap(); + session.commit_downstream(Some(200)).await.unwrap(); + } + if intermediate { + *store.intermediate_gate.lock().unwrap() = Some(gate); + } else { + *store.send_gate.lock().unwrap() = Some(gate); + } + let mut next = Box::pin(session.next_event()); + assert!(futures::poll!(next.as_mut()).is_pending()); + drop(next); + let mut resumed = Box::pin(session.next_event()); + assert!(futures::poll!(resumed.as_mut()).is_pending()); + assert_eq!(provider.contexts.lock().unwrap().len(), 1); + assert_eq!(provider.released_leases.load(Ordering::SeqCst), 0); + release + .send(()) + .expect("the original write still owns its receiver"); + let result = resumed.await; + if retryable { + assert!(result.unwrap().is_some()); + assert_eq!(provider.contexts.lock().unwrap().len(), 2); + assert_eq!(store.state.lock().unwrap().intermediate_failures, 1); + } else { + let Err(EngineError::Provider(error)) = result else { + panic!("original failure must be returned") + }; + assert_eq!(error.kind(), ProviderErrorKind::Transport); + assert_eq!(error.send_state(), UpstreamSendState::Sent); + assert_eq!(provider.contexts.lock().unwrap().len(), 1); + assert_eq!( + store.state.lock().unwrap().finalizations[0].outcome, + ExecutionOutcome::Incomplete + ); + } + drop(session); + assert_eq!( + provider.released_leases.load(Ordering::SeqCst), + if retryable { 2 } else { 1 } + ); + } + }); +} + +#[test] +fn cancelling_a_pending_failure_write_keeps_atomic_usage_and_cost() { + block_on(async { + for intermediate in [false, true] { + let (release, gate) = oneshot::channel(); + let operation = generate_operation(); + let mut failure = bare_atomic_response_failed("charged-failure"); + let mut events = failure.take_atomic_client_events(); + events.extend([ + ProviderEvent::observation( + ProviderResponseObservation::new(UpstreamTransport::new("http_sse").unwrap()) + .with_status_code(429) + .with_request_id(OpaqueUpstreamValue::new("charged-upstream")) + .with_timings(ProviderResponseTimings { + headers_ms: Some(13), + ..ProviderResponseTimings::default() + }), + ), + ProviderEvent::canonical(GatewayEvent::Usage(Usage { + total_tokens: Some(7), + ..Usage::new() + })), + ProviderEvent::canonical(GatewayEvent::ProviderCost( + ProviderReportedCost::from_usd_ticks(25).unwrap(), + )), + ]); + let (coordinator, store, provider) = coordinator(vec![Script::Stream { + account_id: "acct_first", + items: vec![Err(failure.with_atomic_client_events(events))], + }]); + let mut session = coordinator + .start( + model_request(&operation, SystemTime::now() + Duration::from_secs(30)), + operation.clone(), + plan(&operation), + None, + None, + CancellationToken::new(), + ) + .await + .unwrap(); + if intermediate { + *store.intermediate_gate.lock().unwrap() = Some(gate); + } else { + *store.send_gate.lock().unwrap() = Some(gate); + } + let mut next = Box::pin(session.next_event()); + assert!(futures::poll!(next.as_mut()).is_pending()); + drop(next); + session.cancel_and_finalize().await.unwrap(); + assert!( + release.is_canceled(), + "explicit cancellation drops the pending write" + ); + assert_eq!(session.budget_charge().amount_usd.scaled(), 25); + let state = store.state.lock().unwrap(); + assert_eq!(state.finalizations.len(), 1); + assert_eq!(state.finalizations[0].total_tokens, Some(7)); + assert_eq!(state.finalizations[0].cost_ticks, Some(25)); + assert_eq!(state.finalizations[0].send_state, UpstreamSendState::Sent); + assert_eq!(state.finalizations[0].upstream_status_code, Some(429)); + assert_eq!(state.finalizations[0].headers_ms, Some(13)); + assert_eq!( + state.finalizations[0].upstream_request_id.as_deref(), + Some("charged-upstream") + ); + drop(state); + drop(session); + assert_eq!(provider.contexts.lock().unwrap().len(), 1); + assert_eq!(provider.released_leases.load(Ordering::SeqCst), 1); + } + }); +} + #[test] fn rejected_cold_stream_releases_its_lease_before_waiting_for_observation_writes() { block_on(async { diff --git a/backend/crates/gateway-core/tests/engine/middleware/cold.rs b/backend/crates/gateway-core/tests/engine/middleware/cold.rs new file mode 100644 index 000000000..93bb8087b --- /dev/null +++ b/backend/crates/gateway-core/tests/engine/middleware/cold.rs @@ -0,0 +1,206 @@ +//! attempt 中间件在 Core 准入前保持冷流，读取后的错误保留发送事实 + +use super::*; +use gateway_core::{error::ProviderErrorKind, upstream::UpstreamSendState}; + +#[derive(Debug, Clone, Copy)] +enum Action { + RejectBeforeNext, + FaultBeforeNext, + RejectAfterRead, + ReturnAfterRead, +} + +#[derive(Debug)] +struct InspectingPlan(Action); + +impl MiddlewarePlan for InspectingPlan { + fn handle( + &self, + _: MiddlewareContext, + request: MiddlewareRequest, + next: MiddlewareNext, + ) -> BoxFuture<'static, Result> { + let action = self.0; + Box::pin(async move { + match action { + Action::RejectBeforeNext => return Err(MiddlewareError::Rejected), + Action::FaultBeforeNext => return Err(MiddlewareError::Fault), + Action::RejectAfterRead | Action::ReturnAfterRead => {} + } + let response = next.run(request).await?; + let (protocol, status, headers, mut body, envelope) = response.into_parts(); + let first = body.next_frame().await?.expect("upstream first frame"); + if matches!(action, Action::RejectAfterRead) { + return Err(MiddlewareError::Rejected); + } + Ok(MiddlewareResponse::new( + protocol, + status, + headers, + Box::new(InspectedBody { + first: Some(first), + body, + }), + ) + .with_envelope(envelope.expect("host metadata"))) + }) + } +} + +struct InspectedBody { + first: Option, + body: Box, +} + +impl MiddlewareBody for InspectedBody { + fn next_frame(&mut self) -> BoxFuture<'_, Result, MiddlewareError>> { + Box::pin(async move { + if let Some(first) = self.first.take() { + Ok(Some(first)) + } else { + self.body.next_frame().await + } + }) + } + + fn close(self: Box) -> BoxFuture<'static, ()> { + self.body.close() + } +} + +struct DropLease(Arc); + +impl Drop for DropLease { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +fn plan(action: Action) -> FrozenMiddlewarePlan { + FrozenMiddlewarePlan::new( + Arc::new(InspectingPlan(action)), + ExtensionSetReference::new( + ExtensionSetId::new("cold-middleware".to_owned()).unwrap(), + Arc::new(ReadyGeneration), + ), + ) +} + +fn terminal(polls: Arc, released: Arc) -> ProviderMiddlewareTerminal { + Box::new(move |_, _| { + Box::pin(async move { + let metadata = ProviderCallMetadata::new( + ProviderKind::new("openai").unwrap(), + UpstreamModelId::new("gpt-test").unwrap(), + ProviderAccountId::new("acct_middleware").unwrap(), + UpstreamTransport::new("http_sse").unwrap(), + ); + let events = stream::once(async move { + polls.fetch_add(1, Ordering::SeqCst); + Ok(ProviderEvent::canonical_with_wire( + vec![GatewayEvent::Started(ResponseMeta::new( + "response-cold", + "gpt-test", + ))], + ProtocolWireEvent::json( + "openai", + Some("response.created".to_owned()), + json!({"type":"response.created"}), + ) + .unwrap(), + )) + }); + Ok(ProviderStream::new(metadata, events, DropLease(released))) + }) + }) +} + +#[test] +fn inspecting_a_response_waits_for_the_outer_stream_and_keeps_send_facts() { + futures::executor::block_on(async { + for action in [Action::RejectAfterRead, Action::ReturnAfterRead] { + let polls = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicUsize::new(0)); + let mut stream = execute_attempt_middleware( + Some(&plan(action)), + context(), + operation(), + ClientTransport::HttpJson, + terminal(polls.clone(), released.clone()), + ) + .await + .unwrap(); + assert_eq!(polls.load(Ordering::SeqCst), 0); + assert_eq!(released.load(Ordering::SeqCst), 0); + let first = stream.next().await.unwrap(); + match action { + Action::RejectAfterRead => { + let error = first.unwrap_err(); + assert_eq!(error.kind(), ProviderErrorKind::RequestPolicyDenied); + assert_eq!(error.send_state(), UpstreamSendState::Sent); + } + Action::ReturnAfterRead => { + assert!(matches!( + first.unwrap().canonical_facts(), + [GatewayEvent::Started(_)] + )); + } + _ => unreachable!(), + } + assert_eq!(polls.load(Ordering::SeqCst), 1); + drop(stream); + assert_eq!(released.load(Ordering::SeqCst), 1); + } + }); +} + +#[test] +fn rejecting_metadata_drops_the_unactivated_handler_and_account_lease() { + futures::executor::block_on(async { + let polls = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicUsize::new(0)); + let stream = execute_attempt_middleware( + Some(&plan(Action::RejectAfterRead)), + context(), + operation(), + ClientTransport::HttpJson, + terminal(polls.clone(), released.clone()), + ) + .await + .unwrap(); + assert_eq!( + stream.metadata().provider_account_id().as_str(), + "acct_middleware" + ); + drop(stream); + assert_eq!(polls.load(Ordering::SeqCst), 0); + assert_eq!(released.load(Ordering::SeqCst), 1); + }); +} + +#[test] +fn middleware_rejection_before_next_does_not_wait_for_activation() { + futures::executor::block_on(async { + for (action, kind) in [ + ( + Action::RejectBeforeNext, + ProviderErrorKind::RequestPolicyDenied, + ), + (Action::FaultBeforeNext, ProviderErrorKind::Protocol), + ] { + let error = execute_attempt_middleware( + Some(&plan(action)), + context(), + operation(), + ClientTransport::HttpJson, + Box::new(|_, _| panic!("rejected middleware cannot prepare upstream")), + ) + .await + .err() + .expect("prepare rejection"); + assert_eq!(error.kind(), kind); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + } + }); +} diff --git a/backend/crates/gateway-core/tests/engine/middleware/mod.rs b/backend/crates/gateway-core/tests/engine/middleware/mod.rs index c5e74fd21..940eafab6 100644 --- a/backend/crates/gateway-core/tests/engine/middleware/mod.rs +++ b/backend/crates/gateway-core/tests/engine/middleware/mod.rs @@ -1,5 +1,7 @@ //! 请求中间件的改写、交付状态与发布代次持有测试 +mod cold; + use std::num::NonZeroU32; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/backend/crates/gateway-host/src/system_update/operations.rs b/backend/crates/gateway-host/src/system_update/operations.rs index f30b31bfa..b7411a641 100644 --- a/backend/crates/gateway-host/src/system_update/operations.rs +++ b/backend/crates/gateway-host/src/system_update/operations.rs @@ -13,7 +13,7 @@ use gateway_admin::ports::system::{ }; use gateway_core::lifecycle::CancellationToken; use serde::Deserialize; -use tokio::sync::{Mutex as AsyncMutex, broadcast}; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard, broadcast}; use super::archive::extract_release; use super::download::{ @@ -41,7 +41,7 @@ pub struct ProcessSystemOperations { cancellation: CancellationToken, events: Arc, config: Arc, - operation_lock: Arc>, + operation_lock: Arc>>, release_cache: Arc, running_files: Arc>, } @@ -59,24 +59,32 @@ impl ProcessSystemOperations { cancellation, events: Arc::new(UpdateEvents::default()), config: Arc::new(config), - operation_lock: Arc::new(AsyncMutex::new(())), + operation_lock: Arc::new(AsyncMutex::new(None)), release_cache: Arc::new(ReleaseCache::default()), running_files, } } + fn lock_operation(&self) -> Result>, OperationError> { + let operation = Arc::clone(&self.operation_lock) + .try_lock_owned() + .map_err(|_| conflict("system operation is already running"))?; + if operation.is_some() { + return Err(conflict("服务正在重启")); + } + if self.cancellation.is_cancelled() { + return Err(conflict("服务正在关闭")); + } + Ok(operation) + } + fn start_update( &self, target_version: Option, channel: Option, preflight: Arc, ) -> Result { - let operation_lock = Arc::clone(&self.operation_lock) - .try_lock_owned() - .map_err(|_| conflict("system operation is already running"))?; - if self.cancellation.is_cancelled() { - return Err(conflict("服务正在关闭")); - } + let operation_lock = self.lock_operation()?; if let Some(reason) = self.config.update_support_error() { self.events .error_terminal(None, Some("preflight"), reason.clone()); @@ -398,7 +406,7 @@ impl SystemOperations for ProcessSystemOperations { } async fn update_status(&self) -> Result { - if let Ok(_operation) = self.operation_lock.try_lock() { + if let Ok(_operation) = self.lock_operation() { recover_interrupted( &self.config.update_state_file, &self.config.update_lock_file, @@ -418,10 +426,7 @@ impl SystemOperations for ProcessSystemOperations { &self, preflight: Arc, ) -> Result { - let _operation = self - .operation_lock - .try_lock() - .map_err(|_| conflict("system operation is already running"))?; + let _operation = self.lock_operation()?; if let Some(reason) = self.config.update_support_error() { return Err(conflict(reason)); } @@ -496,10 +501,7 @@ impl SystemOperations for ProcessSystemOperations { } async fn restart_candidate(&self) -> Result, OperationError> { - let _operation = self - .operation_lock - .try_lock() - .map_err(|_| conflict("system operation is already running"))?; + let _operation = self.lock_operation()?; self.installed_restart_candidate() } @@ -509,20 +511,20 @@ impl SystemOperations for ProcessSystemOperations { ) -> Result { // 与 update/rollback 互斥：更新替换文件期间触发自重启会让新进程 // 载入半成品产物 - let _operation = self - .operation_lock - .try_lock() - .map_err(|_| conflict("system operation is already running"))?; + let mut operation = self.lock_operation()?; if !self.config.self_restart_enabled { return Err(conflict("self restart is disabled")); } - let _file_lock = OperationFileLock::acquire(&self.config.update_lock_file)?; + let file_lock = OperationFileLock::acquire(&self.config.update_lock_file)?; let candidate = self.installed_restart_candidate()?; let files = candidate .as_ref() .map(|_| ReleaseFiles::installed(&self.config)) .transpose()?; preflight.prepare(candidate).await?; + if self.cancellation.is_cancelled() { + return Err(conflict("服务正在关闭")); + } if let Some(files) = files && files != ReleaseFiles::installed(&self.config)? { @@ -535,6 +537,8 @@ impl SystemOperations for ProcessSystemOperations { "已安排自重启" }; let operation_id = operation_id("restart"); + // 受理后直到旧服务释放，保留重启状态和跨进程锁；延迟关闭只用于送达响应 + *operation = Some(file_lock); let cancellation = self.cancellation.clone(); drop(tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(500)).await; diff --git a/backend/crates/gateway-host/tests/system_update/operations.rs b/backend/crates/gateway-host/tests/system_update/operations.rs index 5387b7ddd..b24b112b2 100644 --- a/backend/crates/gateway-host/tests/system_update/operations.rs +++ b/backend/crates/gateway-host/tests/system_update/operations.rs @@ -38,6 +38,113 @@ async fn restart_should_request_process_restart_inside_docker() { .expect("shutdown requested"); } +#[tokio::test(start_paused = true)] +async fn accepted_restart_excludes_system_operations_until_service_release() { + let fixture = Fixture::new(); + let mut config = fixture.config("http://127.0.0.1:1/repos"); + config.self_restart_enabled = true; + config.deployment_mode = "docker".to_owned(); + let shutdown = CancellationToken::new(); + let service = ProcessSystemOperations::new(shutdown.clone(), config.clone()); + service + .restart(Arc::new(AllowingUpdatePreflight)) + .await + .expect("restart accepted"); + + assert!(!shutdown.is_cancelled(), "response delivery still has time"); + assert_system_operations_rejected(&service, "服务正在重启").await; + assert!(fixture.lock().exists()); + let competing = ProcessSystemOperations::new(CancellationToken::new(), config); + assert_eq!( + competing + .restart(Arc::new(AllowingUpdatePreflight)) + .await + .expect_err("another service cannot replace files during restart") + .kind(), + SystemOperationErrorKind::Conflict + ); + + shutdown.cancelled().await; + assert_system_operations_rejected(&service, "服务正在重启").await; + service + .update_status() + .await + .expect("status remains readable"); + assert!(fixture.lock().exists(), "draining retains the file lock"); + + drop(service); + assert!( + !fixture.lock().exists(), + "service release clears the file lock" + ); +} + +#[tokio::test] +async fn shutdown_rejects_new_system_operations() { + let fixture = Fixture::new(); + let mut config = fixture.config("http://127.0.0.1:1/repos"); + config.self_restart_enabled = true; + config.deployment_mode = "docker".to_owned(); + let shutdown = CancellationToken::new(); + let service = ProcessSystemOperations::new(shutdown.clone(), config); + shutdown.cancel(); + + assert_system_operations_rejected(&service, "服务正在关闭").await; + assert!(!fixture.lock().exists()); +} + +#[tokio::test] +async fn shutdown_during_restart_preflight_does_not_schedule_replacement() { + struct CancellingPreflight(CancellationToken); + #[async_trait::async_trait] + impl gateway_admin::ports::system::SystemRestartPreflight for CancellingPreflight { + async fn prepare( + &self, + _: Option, + ) -> Result<(), SystemOperationError> { + self.0.cancel(); + Ok(()) + } + } + + let fixture = Fixture::new(); + let mut config = fixture.config("http://127.0.0.1:1/repos"); + config.self_restart_enabled = true; + config.deployment_mode = "docker".to_owned(); + let shutdown = CancellationToken::new(); + let service = ProcessSystemOperations::new(shutdown.clone(), config); + let error = service + .restart(Arc::new(CancellingPreflight(shutdown))) + .await + .expect_err("shutdown won during preflight"); + + assert_eq!(error.kind(), SystemOperationErrorKind::Conflict); + assert_eq!(error.message(), "服务正在关闭"); + assert!(!fixture.lock().exists()); +} + +async fn assert_system_operations_rejected(service: &ProcessSystemOperations, reason: &str) { + for result in [ + service.restart(Arc::new(AllowingUpdatePreflight)).await, + service.rollback(Arc::new(AllowingUpdatePreflight)).await, + service + .perform_test_update(Some(TARGET_VERSION.to_owned())) + .await, + ] { + let error = result.expect_err("system operation must not be accepted"); + assert_eq!(error.kind(), SystemOperationErrorKind::Conflict); + assert_eq!(error.message(), reason); + } + assert_eq!( + service + .restart_candidate() + .await + .expect_err("no restart plan") + .message(), + reason + ); +} + #[tokio::test] async fn restart_should_spawn_replacement_before_shutdown_outside_docker() { let fixture = Fixture::new(); diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs index 6a02c9ce5..ff5f9b9c6 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs @@ -97,18 +97,14 @@ impl PluginCommand { .await .map_err(rpc_failure)?; if reply.payload.len() > limits.maximum_buffered_body_bytes { - self.session.invalid_response(Stage::CommandLine); return Err("结果信封无效"); } - let result: CommandResult = serde_json::from_slice(&reply.payload).map_err(|_| { - self.session.invalid_response(Stage::CommandLine); - "结果数据无效" - })?; + let result: CommandResult = + serde_json::from_slice(&reply.payload).map_err(|_| "结果数据无效")?; if result.stdout.len() > MAXIMUM_OUTPUT_BYTES || result.stderr.len() > MAXIMUM_OUTPUT_BYTES || result.accounts.len() > 32 { - self.session.invalid_response(Stage::CommandLine); return Err("结果大小或账号范围无效"); } if result.exit_code == 0 { diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/frontend_authentication.rs b/backend/crates/gateway-plugin/runtime/src/adapter/frontend_authentication.rs index 3bdfaa88b..fe9c60df8 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/frontend_authentication.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/frontend_authentication.rs @@ -214,11 +214,8 @@ impl FrontendAuthenticationPlan for PluginFrontendAuthenticationPlan { ) .await .map_err(|_| FrontendAuthenticationError)?; - let result: FrontendAuthenticationResult = serde_json::from_slice(&reply.payload) - .map_err(|_| { - session.invalid_response(Stage::Authentication); - FrontendAuthenticationError - })?; + let result: FrontendAuthenticationResult = + serde_json::from_slice(&reply.payload).map_err(|_| FrontendAuthenticationError)?; match result { FrontendAuthenticationResult::Authenticated { principal } if valid_principal(&principal) => @@ -226,7 +223,6 @@ impl FrontendAuthenticationPlan for PluginFrontendAuthenticationPlan { Ok(FrontendAuthenticationDecision::Authenticated { principal }) } FrontendAuthenticationResult::Authenticated { .. } => { - session.invalid_response(Stage::Authentication); Err(FrontendAuthenticationError) } FrontendAuthenticationResult::NotMatched {} => { diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/management/callback.rs b/backend/crates/gateway-plugin/runtime/src/adapter/management/callback.rs index 552fc0782..06141929d 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/management/callback.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/management/callback.rs @@ -198,11 +198,7 @@ impl ManagementEntry { .map_err(|_| { AdminError::unavailable("插件登录回调未完成；state 已消费，请重新发起登录") })?; - super::decode_response(reply, &descriptor.response_content_types, limits).inspect_err( - |_| { - self.session.invalid_response(Stage::PublicManagement); - }, - ) + super::decode_response(reply, &descriptor.response_content_types, limits) } } diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/management/mod.rs b/backend/crates/gateway-plugin/runtime/src/adapter/management/mod.rs index 28a31bb97..14df27f5a 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/management/mod.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/management/mod.rs @@ -112,9 +112,7 @@ impl ManagementEntry { .map_err(|_| { AdminError::unavailable("插件管理调用未完成；副作用可能已发生，请查询后再重试") })?; - decode_response(reply, &route.response_content_types, limits).inspect_err(|_| { - self.session.invalid_response(Stage::Management); - }) + decode_response(reply, &route.response_content_types, limits) } } diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/http.rs b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/http.rs index 0ccdbc09d..e645e9c4b 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/http.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/http.rs @@ -66,24 +66,21 @@ pub(super) async fn invoke( .decode_response(Stage::Http, std::mem::take(&mut stream.initial.result)) .map_err(|_| MiddlewareError::InvalidState)?; let payload = std::mem::take(&mut stream.initial.payload); - let invalid = || { - ports.session.invalid_response(Stage::Http); - MiddlewareError::InvalidState - }; if response.session { - let status = invocation.session_status().ok_or_else(invalid)?; + let status = invocation + .session_status() + .ok_or(MiddlewareError::InvalidState)?; if response.status != status.as_u16() || response.response.is_none() || !matches!(response.body, wire::Body::Empty) || !payload.is_empty() { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } let mut response = invocation .complete(response, payload, None) .await - .map_err(|_| invalid())?; - let session = ports.session.clone(); + .map_err(|_| MiddlewareError::InvalidState)?; let task = Box::pin(async move { let _binding = _binding; tokio::select! { @@ -94,7 +91,7 @@ pub(super) async fn invoke( match stream.next().await { Ok(None) => Ok(()), Ok(Some(_)) => { - let error = session.invalid_response(Stage::Http); + let error = crate::RpcError::InvalidResponse(Stage::Http, None); Err(session_error(error, &entry.instance_id)) } Err(error) => Err(session_error(error, &entry.instance_id)), @@ -108,23 +105,19 @@ pub(super) async fn invoke( return Ok(response); } let body = if matches!(response.body, wire::Body::Stream) { - let body = StreamBody::new(stream::try_unfold((stream, invocation.clone(), context.cancellation, ports.session.clone()), |(mut stream, invocation, cancellation, session)| async move { + let body = StreamBody::new(stream::try_unfold((stream, invocation.clone(), context.cancellation), |(mut stream, invocation, cancellation)| async move { let chunk = tokio::select! { biased; () = cancellation.cancelled() => return Err(Box::new(MiddlewareError::Fault) as Box), chunk = stream.next() => chunk.map_err(|error| Box::new(crate::callback::error::rpc_middleware(error)) as Box)?, }; let Some(chunk) = chunk else { return Ok(None); }; - let invalid = || { - session.invalid_response(Stage::Http); - MiddlewareError::InvalidState - }; let frame = match chunk.split_first() { Some((0, data)) => Frame::data(bytes::Bytes::copy_from_slice(data)), - Some((1, data)) => Frame::trailers(headers(serde_json::from_slice(data).map_err(|_| invalid())?).map_err(|_| invalid())?), - _ => return Err(Box::new(invalid()) as _), + Some((1, data)) => Frame::trailers(headers(serde_json::from_slice(data).map_err(|_| MiddlewareError::InvalidState)?).map_err(|_| MiddlewareError::InvalidState)?), + _ => return Err(Box::new(MiddlewareError::InvalidState) as _), }; - Ok(Some((frame, (stream, invocation, cancellation, session)))) + Ok(Some((frame, (stream, invocation, cancellation)))) })).boxed_unsync(); Some(body) } else { @@ -135,14 +128,14 @@ pub(super) async fn invoke( .map_err(crate::callback::error::rpc_middleware)? .is_some() { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } None }; invocation .complete(response, payload, body) .await - .map_err(|_| invalid()) + .map_err(|_| MiddlewareError::InvalidState) } fn session_error(error: crate::RpcError, instance_id: &str) -> core::upgrade::SessionError { diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/request.rs b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/request.rs index d38d12e66..f8fb59254 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/request.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/request.rs @@ -130,7 +130,6 @@ pub(super) async fn invoke( .map(|account| account.as_str().to_owned()), headers, }; - let stage = call_context.stage; let mut stream = tokio::select! { biased; () = context.cancellation().cancelled() => return Err(MiddlewareError::Fault), @@ -154,7 +153,7 @@ pub(super) async fn invoke( Ok(response) => response, Err(_) => { drop(stream); - return recover_invalid(&entry, &invocation, stage).await; + return recover_invalid(&entry, &invocation).await; } }; if let Some(error) = invocation.take_downstream_error() { @@ -163,26 +162,23 @@ pub(super) async fn invoke( let completion = match invocation.resolve_response(response).await { Ok(completion) => completion, Err(error) => { - invocation_ports.session.invalid_response(stage); drop(stream); return Err(error); } }; let body: Box = match completion.body { MiddlewareCompletionBody::PassThrough(body) => { - drain_empty_stream(stream, &invocation, &invocation_ports.session, stage).await?; + drain_empty_stream(stream, &invocation).await?; body } MiddlewareCompletionBody::Empty => { - drain_empty_stream(stream, &invocation, &invocation_ports.session, stage).await?; + drain_empty_stream(stream, &invocation).await?; Box::new(EmptyMiddlewareBody) } MiddlewareCompletionBody::Stream { framing, downstream, } => Box::new(PluginMiddlewareBody { - session: Arc::clone(&invocation_ports.session), - stage, stream: Some(stream), framing, terminal_seen: false, @@ -243,11 +239,7 @@ async fn recover_rpc( async fn recover_invalid( entry: &MiddlewareEntry, invocation: &MiddlewareInvocation, - stage: Stage, ) -> Result { - if let Some(ports) = &entry.invocation { - ports.session.invalid_response(stage); - } if entry.failure_policy == PluginFailurePolicy::Delegate && let Some((next, request)) = invocation.take_delegate() { @@ -265,15 +257,10 @@ async fn recover_invalid( async fn drain_empty_stream( mut stream: RpcStream, invocation: &MiddlewareInvocation, - session: &crate::RpcSession, - stage: Stage, ) -> Result<(), MiddlewareError> { match stream.next().await { Ok(None) => invocation.take_downstream_error().map_or(Ok(()), Err), - Ok(Some(_)) => { - session.invalid_response(stage); - Err(MiddlewareError::InvalidState) - } + Ok(Some(_)) => Err(MiddlewareError::InvalidState), Err(error) => invocation .take_downstream_error() .map_or_else(|| Err(crate::callback::error::rpc_middleware(error)), Err), @@ -293,8 +280,6 @@ impl MiddlewareBody for EmptyMiddlewareBody { } struct PluginMiddlewareBody { - session: Arc, - stage: Stage, stream: Option, framing: MiddlewareFraming, terminal_seen: bool, @@ -311,10 +296,6 @@ struct PluginMiddlewareBody { impl MiddlewareBody for PluginMiddlewareBody { fn next_frame(&mut self) -> BoxFuture<'_, Result, MiddlewareError>> { Box::pin(async move { - let invalid = || { - self.session.invalid_response(self.stage); - MiddlewareError::InvalidState - }; if let Some(error) = take_error(&self.downstream_error) { self.stream.take(); return Err(error); @@ -340,15 +321,16 @@ impl MiddlewareBody for PluginMiddlewareBody { None => false, } { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } return take_error(&self.downstream_error).map_or(Ok(None), Err); }; if self.terminal_seen { self.stream.take(); - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } - let wire_frame = MiddlewareBodyFrame::decode(&chunk).map_err(|_| invalid())?; + let wire_frame = MiddlewareBodyFrame::decode(&chunk) + .map_err(|_| MiddlewareError::InvalidState)?; let source_id = wire_frame.source_id(); let disposition = wire_frame.disposition(); let frame = match (&self.downstream, disposition) { @@ -361,7 +343,7 @@ impl MiddlewareBody for PluginMiddlewareBody { .with_transformed(true), ), (Some(_), MiddlewareBodyDisposition::Standalone) | (None, _) => { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } (Some(downstream), MiddlewareBodyDisposition::Only) | (Some(downstream), MiddlewareBodyDisposition::Drop) => { @@ -369,7 +351,7 @@ impl MiddlewareBody for PluginMiddlewareBody { || source_id != self.next_source_id.saturating_add(1) || wire_frame.payload.len() > MAXIMUM_MAPPED_BYTES_PER_SOURCE { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } self.next_source_id = source_id; downstream @@ -379,14 +361,14 @@ impl MiddlewareBody for PluginMiddlewareBody { Bytes::from(wire_frame.payload), ) .await - .map_err(|_| invalid())? + .map_err(|_| MiddlewareError::InvalidState)? } (Some(downstream), MiddlewareBodyDisposition::First) => { if self.active_source.is_some() || source_id != self.next_source_id.saturating_add(1) || wire_frame.payload.len() > MAXIMUM_MAPPED_BYTES_PER_SOURCE { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } self.active_source = Some(source_id); self.active_output_count = 1; @@ -398,22 +380,22 @@ impl MiddlewareBody for PluginMiddlewareBody { Bytes::from(wire_frame.payload), ) .await - .map_err(|_| invalid())? + .map_err(|_| MiddlewareError::InvalidState)? } (Some(downstream), MiddlewareBodyDisposition::More) => { self.active_output_count = self .active_output_count .checked_add(1) - .ok_or_else(invalid)?; + .ok_or(MiddlewareError::InvalidState)?; self.active_output_bytes = self .active_output_bytes .checked_add(wire_frame.payload.len()) - .ok_or_else(invalid)?; + .ok_or(MiddlewareError::InvalidState)?; if self.active_source != Some(source_id) || self.active_output_count > MAXIMUM_MAPPED_FRAMES_PER_SOURCE || self.active_output_bytes > MAXIMUM_MAPPED_BYTES_PER_SOURCE { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } downstream .resolve_source_output( @@ -422,22 +404,22 @@ impl MiddlewareBody for PluginMiddlewareBody { Bytes::from(wire_frame.payload), ) .await - .map_err(|_| invalid())? + .map_err(|_| MiddlewareError::InvalidState)? } (Some(downstream), MiddlewareBodyDisposition::Last) => { self.active_output_count = self .active_output_count .checked_add(1) - .ok_or_else(invalid)?; + .ok_or(MiddlewareError::InvalidState)?; self.active_output_bytes = self .active_output_bytes .checked_add(wire_frame.payload.len()) - .ok_or_else(invalid)?; + .ok_or(MiddlewareError::InvalidState)?; if self.active_source != Some(source_id) || self.active_output_count > MAXIMUM_MAPPED_FRAMES_PER_SOURCE || self.active_output_bytes > MAXIMUM_MAPPED_BYTES_PER_SOURCE { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } self.active_source = None; self.active_output_count = 0; @@ -450,7 +432,7 @@ impl MiddlewareBody for PluginMiddlewareBody { Bytes::from(wire_frame.payload), ) .await - .map_err(|_| invalid())? + .map_err(|_| MiddlewareError::InvalidState)? } }; if disposition == MiddlewareBodyDisposition::Drop { @@ -460,7 +442,7 @@ impl MiddlewareBody for PluginMiddlewareBody { } continue; } - let mut frame = frame.ok_or_else(invalid)?; + let mut frame = frame.ok_or(MiddlewareError::InvalidState)?; // 只有宿主来源事件本就没有 wire 时，空 payload 才是事实帧 let facts_only = frame.bytes().is_empty() && frame @@ -471,7 +453,7 @@ impl MiddlewareBody for PluginMiddlewareBody { && !facts_only && !valid_plugin_frame(self.framing, frame.bytes()) { - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } if self.pending_transformed { frame = frame.with_transformed(true); @@ -485,14 +467,11 @@ impl MiddlewareBody for PluginMiddlewareBody { } { self.stream.take(); - return Err(invalid()); + return Err(MiddlewareError::InvalidState); } self.terminal_seen = true; let terminal = stream.next().await; self.stream.take(); - if matches!(&terminal, Ok(Some(_))) { - self.session.invalid_response(self.stage); - } validate_terminal_end(terminal, &self.downstream_error)?; } return Ok(Some(frame)); diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/websocket.rs b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/websocket.rs index f1d5b6568..1a66d00f8 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/middleware/websocket.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/middleware/websocket.rs @@ -70,15 +70,11 @@ pub(super) async fn invoke( .session .decode_response(Stage::WebSocket, reply.result) .map_err(|_| MiddlewareError::InvalidState)?; - let invalid = || { - ports.session.invalid_response(Stage::WebSocket); - MiddlewareError::InvalidState - }; match output { Some(message) => invocation .decode(message, reply.payload) .map(Some) - .map_err(|_| invalid()), + .map_err(|_| MiddlewareError::InvalidState), None => Ok(None), } } diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs index c5f2389e4..e3117c651 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs @@ -35,17 +35,6 @@ pub(super) struct DecodedEvent { pub(super) continuation: Option, } -pub(super) enum EventDecodeError { - Invalid(ProviderError), - Upstream(ProviderError), -} - -impl From for EventDecodeError { - fn from(error: ProviderError) -> Self { - Self::Invalid(error) - } -} - impl EventDecoder { pub(super) fn decode( &mut self, @@ -55,11 +44,11 @@ impl EventDecoder { account: &dyn UpstreamAccountConnection, sent: UpstreamSendState, started_at: std::time::Instant, - ) -> Result { + ) -> Result { let message = UpstreamAdapterEvent::decode(payload) .map_err(|source| invalid(sent).with_source(source))?; if self.completed || message.event.facts.len() > 64 { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } let wire = message .event @@ -79,7 +68,7 @@ impl EventDecoder { } if let Some(failure) = message.failure { if !message.event.facts.is_empty() || message.continuation.is_some() { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } let mut events = Vec::new(); if has_observation { @@ -88,9 +77,7 @@ impl EventDecoder { if let Some(wire) = wire { events.push(ProviderEvent::wire(wire)); } - return Err(EventDecodeError::Upstream( - failure_error(failure, sent)?.with_atomic_client_events(events), - )); + return Err(failure_error(failure, sent)?.with_atomic_client_events(events)); } let mut facts = Vec::new(); for fact in message.event.facts { @@ -104,7 +91,7 @@ impl EventDecoder { CanonicalEvent::ContentAdded { index, kind } => { self.contents += 1; if self.contents > 4096 { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } GatewayEvent::ContentAdded(ContentItem::new( index, @@ -146,7 +133,7 @@ impl EventDecoder { .as_ref() .is_some_and(|name| name.len() > 512 || name.chars().any(char::is_control)) { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } GatewayEvent::ToolCallDelta(ToolCallDelta { content_index: index, @@ -170,7 +157,7 @@ impl EventDecoder { .flatten() .any(|count| count > i64::MAX as u64) { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } let usage = Usage { input_tokens: usage.input_tokens, @@ -188,7 +175,7 @@ impl EventDecoder { CanonicalEvent::Completed { id, model, reason } => { validate_model(model.as_deref(), sent)?; if self.response_id.as_deref() != Some(id.as_str()) { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } if let Some(cost) = account.calculate_cost(observation.service_tier(), &self.usage) @@ -221,7 +208,7 @@ impl EventDecoder { facts.push(fact); } if message.continuation.is_some() && !self.completed { - return Err(invalid(sent).into()); + return Err(invalid(sent)); } observation = super::observation::observe_output(observation, &facts, wire.is_some(), started_at); diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs index 541c86e0a..d5eb382b4 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use super::{ PluginUpstreamAdapter, - event::{DecodedEvent, EventDecodeError, EventDecoder}, + event::{DecodedEvent, EventDecoder}, failure::{fault_error, invalid, rpc_error}, }; use crate::{ @@ -94,27 +94,15 @@ impl Execution { biased; () = cancellation.cancelled() => return Err(ProviderError::new(ProviderErrorKind::Cancelled, active.managed.send_state.get())), result = active.stream.next() => result.map_err(|error| rpc_error(error, active.managed.send_state.get()))?, - }.ok_or_else(|| { - self.adapter.session.invalid_response(Stage::Upstream); - invalid(active.managed.send_state.get()) - })?; - let mut decoded = self - .decoder - .decode( - &chunk, - &self.adapter.declaration.protocol, - self.adapter.declaration.transport.as_str(), - self.invocation.account.as_ref(), - active.managed.send_state.get(), - self.invocation.context.timing_started_at(), - ) - .map_err(|error| match error { - EventDecodeError::Invalid(error) => { - self.adapter.session.invalid_response(Stage::Upstream); - error - } - EventDecodeError::Upstream(error) => error, - })?; + }.ok_or_else(|| invalid(active.managed.send_state.get()))?; + let mut decoded = self.decoder.decode( + &chunk, + &self.adapter.declaration.protocol, + self.adapter.declaration.transport.as_str(), + self.invocation.account.as_ref(), + active.managed.send_state.get(), + self.invocation.context.timing_started_at(), + )?; if self.decoder.completed { // 先确认 RPC 正常终结，再发布唯一 Completed；否则 Core 不会继续 poll 尾部错误 let terminal = async { @@ -124,7 +112,6 @@ impl Execution { result = active.stream.next() => result.map_err(|error| rpc_error(error, active.managed.send_state.get()))?, }; if tail.is_some() { - self.adapter.session.invalid_response(Stage::Upstream); return Err(invalid(active.managed.send_state.get())); } self.decoder.finish(active.managed.send_state.get())?; diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs index e6e0d49cb..c3b9e03ea 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs @@ -199,11 +199,6 @@ pub struct RpcSession { } impl RpcSession { - /// 响应合同错误只结束所属调用，不影响共享传输与其他在途调用 - pub(crate) fn invalid_response(&self, stage: Stage) -> RpcError { - RpcError::InvalidResponse(stage, None) - } - pub(crate) fn decode_response( &self, stage: Stage, diff --git a/backend/crates/gateway-plugin/sdk/README.md b/backend/crates/gateway-plugin/sdk/README.md index 7f1c03289..823400ac3 100644 --- a/backend/crates/gateway-plugin/sdk/README.md +++ b/backend/crates/gateway-plugin/sdk/README.md @@ -1,15 +1,9 @@ # Gateway Plugin SDK 用于编写 Codex Proxy 插件的 Rust 合同与可选会话辅助。插件只依赖本包，不依赖宿主 Core、Admin、Store 或 Host。 -SDK 处于实验阶段；能力是否可用取决于宿主支持、清单声明与实际注册， -而不只是枚举中存在该名称 +SDK 包版本为 `0.1.0`，独立于网关版本，处于实验阶段。能力是否可用取决于宿主支持、清单声明与实际注册 -| 版本 | 当前值 | 用途 | -| --- | --- | --- | -| SDK 包 | `0.1.0` | Rust 开发依赖，独立于网关版本 | -| `manifestVersion` | `2` | 安装清单格式 | -| `package.protocolVersion` | `5` | 宿主与插件的进程通信 | -| `contributes.*.version` | `middleware` 为 `5`，`scheduler` 为 `3`，`upstream_adapter` 为 `4`，其余见声明 | 单项能力合同，见[扩展项声明](docs/manifest.md#扩展项简写) | +清单、进程协议和各项能力分别版本化，当前版本与声明规则统一见[插件清单](docs/manifest.md) 开发顺序：编写[清单](docs/manifest.md) → 实现[能力处理器](docs/capabilities.md) → 用 [CLI](../../../apps/plugin-cli/README.md) 打包 → [安装、配置与启用](../../../../docs/plugins.md)。完整可运行示例位于独立仓库 @@ -51,12 +45,7 @@ use gateway_plugin_sdk::client::{read_frame, write_frame}; 默认 feature 不引入异步运行时。`io` 除帧编解码外，还提供 `PluginBuilder`、 `PluginSession::accept/run` 和 `PluginHandler`。组合插件从作者清单建立类型化方法，SDK 自动生成 `plugin.register`；作者不需要复制贡献项或手写方法名、阶段和敏感载荷编解码。 -会话管理握手、有界并发、父调用关联、回调、流信用、取消、排空与关闭。 - -`SessionConfig.maximum_calls` 默认 256，限制包含下游等待和响应流在内的在途调用；`maximum_callbacks` 默认 64， -限制普通宿主回调。`next` 使用独立回调预算，上限为 `maximum_calls`，不挤占普通回调容量。 -宿主负责执行与准入调度，等待与处理独立计时；默认预算与过载行为见[插件使用](../../../../docs/plugins.md#故障处理)。 -插件显式覆盖配置时需与宿主协调在途预算；较小的插件侧上限会提前拒绝新调用 +会话管理握手、有界并发、父调用关联、回调、流信用、取消、排空与关闭，容量与计时见[并发与期限](#并发与期限) `ResponseStream::channel` 提供有界流生产入口，`ResponseStream::pull` 按消费进度驱动生产者；`PluginCall` 的宿主客户端只代表当前父调用， 不能把一个调用的资源误用到另一个调用。插件程序自行选择运行时和启动入口，不会启动网关。 @@ -86,6 +75,22 @@ sequenceDiagram P-->>H: 结果或错误；流以 End 结束 ``` +### 并发与期限 + +每个宿主插件会话默认允许 64 个执行中的 RPC 调用、64 个初次排队调用和 256 个总在途上下文。 +初次排队和等待 `next` 后的恢复排队各限 30 秒，不扣插件自身默认上限 120 秒的处理预算。 +恢复等待占总在途名额，不占初次排队名额。 +等待 `next` 时释放执行名额并暂停处理预算，返回后重新取得名额，继续同一次调用； +已返回响应头的流继续占用在途预算，不占首结果执行名额。具体组合行为见[洋葱中间件](docs/capabilities.md#洋葱中间件) + +插件侧 `SessionConfig.maximum_calls` 默认 256，限制包含下游等待和响应流在内的在途调用； +`maximum_callbacks` 默认 64，限制普通宿主回调。宿主普通回调的默认上限同样为 64， +`next` 使用总在途上限对应的独立回调预算，不挤占普通回调容量 + +实际容量由宿主预算与 SDK 会话配置共同限制，较小的插件侧上限会提前拒绝新调用。 +达到容量或等待上限时，中间件返回过载错误；HTTP 响应为 503（`middleware_capacity_unavailable`）， +不跳过插件或重放请求。调整 SDK 会话配置需重新构建插件包，容量应结合插件耗时、正文和响应流的资源占用评估 + ### 帧与会话 `CallContext.timeout_ms` 限定初始处理期限。宿主对由流或连接 owner 管理的调用设置 `resource_stream`： @@ -117,6 +122,10 @@ SDK 帧读写器自动拆装这些字段，不拼接复制二进制正文。`Mes ## 本地验证 +以下命令面向宿主仓库中的 SDK 维护。独立插件的构建、安装与能力验收见[开发技能](../../../../.agents/skills/cpr-plugin-dev/SKILL.md)及所用示例 + +### 生成服务合同 + 设置与账号自动重置服务的 SDK 类型和操作由宿主领域类型及[操作声明](../../gateway-admin/src/public_service)生成，SDK 构建不依赖宿主源码。 修改宿主合同后，在仓库根目录更新生成文件；Admin 合同测试会检查生成结果是否同步： @@ -128,7 +137,7 @@ cargo +1.99.1 fmt --all --manifest-path backend/Cargo.toml 用量服务合同由 Admin 与 SDK 分别持有，`sdk_wire_types_match_host_contracts` 检查数值、明细、聚合、查询和计价类型一致性；修改合同需同步两侧类型与 [使用说明](docs/usage-data.md) -SDK 自身的验证命令： +### 验证命令 ```bash RUST_MIN_STACK=16777216 cargo +1.99.1 test --manifest-path backend/Cargo.toml -p gateway-plugin-sdk --no-default-features --locked diff --git a/backend/crates/gateway-plugin/sdk/docs/capabilities.md b/backend/crates/gateway-plugin/sdk/docs/capabilities.md index fb92115f1..3ab4634e2 100644 --- a/backend/crates/gateway-plugin/sdk/docs/capabilities.md +++ b/backend/crates/gateway-plugin/sdk/docs/capabilities.md @@ -6,7 +6,7 @@ | 开发任务 | 入口 | | --- | --- | -| 组合处理器 | [作者入口](#类型化作者入口) · [能力与方法](#能力与方法) | +| 组合处理器 | [作者入口](#类型化作者入口) · [请求上下文](#请求上下文) · [能力与方法](#能力与方法) | | 使用宿主资源 | [基础事实](#基础事实) · [额度刷新](#额度观测刷新) · [账号](#账号与凭据) · [自有资源](#自有资源与维护) · [Key 预算](#client-key-预算) · [模型](#key模型与模型调用) · [网络](#网络) | | 扩展请求链 | [中间件](#洋葱中间件) · [上游适配器](upstream-adapters.md) · [模型目录](#模型目录) · [重试](#重试决策) · [路由、调度与观察](#路由调度与观察) · [入口认证](#数据面入口认证) | | 管理与交互 | [状态、日志与迁移](#状态日志与迁移) · [管理页面](#管理页面公开入口与-cli) · [页面宿主桥](#页面宿主桥-v2) · [命令行](#命令行) | @@ -273,10 +273,9 @@ attempt、路由、调度、重试和上游适配器使用执行冻结预算， `PluginBuilder::middleware` 与 `MiddlewarePlugin::new` 共用 `MiddlewareInput` / `MiddlewareOutput`。 `next.run(input)` 消费自身，最多执行一次；结果按相反顺序经过外层。重试或独立子请求通过宿主执行服务发起 -调用上下文的 `timeout_ms` 限制插件自身处理时间，默认上限为 120 秒。等待宿主 `next` 时暂停这份预算， -宿主等待期间归还执行名额，返回后先重新取得名额再交付结果或错误，插件继续同一次调用。 -初次准入和恢复排队各有独立等待期限，默认 30 秒，不扣插件处理预算；恢复等待受总在途上限约束，不占初次准入队列。 -入站处理与返回处理合计使用同一预算，普通宿主回调不会暂停计时。 +调用上下文的 `timeout_ms` 限制插件自身处理时间，入站与返回处理合计使用同一预算。 +等待 `next` 时暂停该预算并释放执行名额；恢复后继续同一次调用，普通宿主回调不会暂停计时。 +排队、在途容量与过载行为见 [SDK 并发与期限](../README.md#并发与期限)。 请求总期限、下游操作超时和取消信号独立生效，客户端取消或插件退出会结束等待并回收资源 HTTP 总入口统一包裹原生路由，`dispatch_http` 复用同一路由与组合能力；无需为每个 HTTP 接口登记新的挂载点。 @@ -403,9 +402,8 @@ Responses 终端重新解码插件正文时读取改写后的正文上限；atte 更早的改写使用上述 HTTP 请求的 `settings`。`request.head.settings_sources` 使用相同的来源结构，`execution.input` 表示进入当前 Key 作用域时的有效值。 原生 Responses WebSocket 每轮请求读取当前宿主快照，重新认证并解析 Key 策略，同时保留握手阶段的显式改写；已开始的执行保持自己的冻结值 -中间件可读取和改写完整请求、响应 headers、正文与响应帧，不要求 -`requests` 权限，也不按字段名称过滤认证、Cookie、会话或连接头。安装者承担插件读取与修改这些数据的风险。 -类型、HTTP 格式、大小及资源生命周期仍需符合合同 +中间件可读取和改写完整请求、响应 headers、正文与响应帧，也不按字段名称过滤认证、Cookie、会话或连接头。 +该能力遵循[完整信任](manifest.md#完整信任)边界；类型、HTTP 格式、大小及资源生命周期仍需符合合同 默认使用 `MiddlewareRequestBody::Preserve` 保持正文；`replace_body(Vec::new())` 明确清空。 Header 修改采用增量 remove/append，保留合法多值。`MiddlewareBody::map_frames` 复用会话的流控和唯一终态 diff --git a/backend/crates/gateway-plugin/sdk/docs/manifest.md b/backend/crates/gateway-plugin/sdk/docs/manifest.md index 96dda0378..e57eebb9d 100644 --- a/backend/crates/gateway-plugin/sdk/docs/manifest.md +++ b/backend/crates/gateway-plugin/sdk/docs/manifest.md @@ -23,16 +23,16 @@ 宿主实际开放的合同由 [plugin-host-compatibility.json](../../runtime/plugin-host-compatibility.json) 声明，SDK 的 `Capability::contract_versions()` 只表示 SDK 能描述的行为版本，不能代替宿主支持检查 -| 版本字段 | 对应合同 | -| --- | --- | -| `manifestVersion` / `manifest_schema_versions` | 插件清单格式 | -| `package.protocolVersion` / `protocol_versions` | 插件进程 RPC 封装，不是 OpenAI 或 xAI 的业务协议 | -| `contributes..version` / `capabilities[].versions` | 指定扩展能力的行为合同 | -| 宿主声明的 `schema_version` | 宿主兼容声明自身的格式 | +| 版本字段 | 当前值 | 对应合同 | +| --- | --- | --- | +| `manifestVersion` / `manifest_schema_versions` | `2` | 插件清单格式 | +| `package.protocolVersion` / `protocol_versions` | `5` | 插件进程 RPC 封装，不是 OpenAI 或 xAI 的业务协议 | +| `contributes..version` / `capabilities[].versions` | 见[扩展项简写](#扩展项简写) | 指定扩展能力的行为合同 | +| 宿主声明的 `schema_version` | `2` | 宿主兼容声明自身的格式 | `capabilities` 使用扩展能力标识；Provider、模型和账号 ID 不属于这个集合 -清单与宿主兼容声明的格式版本为 `2`，进程协议版本为 `4`；清单不接受 `permissions` 字段 +清单不接受 `permissions` 字段；插件按[完整信任](#完整信任)运行 RPC v5 的帧布局、编码与会话生命周期见[通信边界](../README.md#通信边界)。插件使用 SDK 构建，并由 CLI 生成包含协议版本的安装包 @@ -49,7 +49,17 @@ RPC v5 的帧布局、编码与会话生命周期见[通信边界](../README.md# ## 扩展项简写 -作者声明可以省略 `id` 和固定阶段。版本字段省略时为 `1`，适用于 `model_catalog`、`command_line` 和 `maintenance`；其余能力显式声明当前版本：`middleware` 为 `5`，`scheduler` 为 `3`，`upstream_adapter` 为 `4`，认证、路由、重试、观察和管理能力为 `2`： +作者声明可以省略 `id` 和固定阶段。版本字段省略时为 `1`，其余版本必须显式声明： + +| 能力 | 当前版本 | +| --- | --- | +| `middleware` | `5` | +| `scheduler` | `3` | +| `upstream_adapter` | `4` | +| `frontend_authentication`、`model_router`、`retry_policy`、`observer`、`management` | `2` | +| `model_catalog`、`command_line`、`maintenance` | `1` | + +例如，仅声明管理页面、命令行和模型请求中间件： ```json { diff --git a/backend/crates/gateway-plugin/sdk/docs/upstream-adapters.md b/backend/crates/gateway-plugin/sdk/docs/upstream-adapters.md index bfebde7a2..a603fb9f5 100644 --- a/backend/crates/gateway-plugin/sdk/docs/upstream-adapters.md +++ b/backend/crates/gateway-plugin/sdk/docs/upstream-adapters.md @@ -1,5 +1,7 @@ # 受管上游适配器 +[返回 SDK](../README.md) · [清单声明](manifest.md#扩展项简写) · [能力与回调](capabilities.md) + `upstream_adapter` v4 是现有生成请求洋葱链的终端，只挂到内置 `openai` / `xai`，不替换独立图像或压缩端点。 插件定义 Base URL、业务路径、请求编码和响应解析；宿主继续持有身份、ACL、选号、租约、凭据、代理、重试和账本。 普通请求或响应加工使用 [middleware](capabilities.md#洋葱中间件) diff --git a/backend/crates/gateway-plugin/sdk/docs/usage-data.md b/backend/crates/gateway-plugin/sdk/docs/usage-data.md index 7dd96cfd8..7f1b5ad7c 100644 --- a/backend/crates/gateway-plugin/sdk/docs/usage-data.md +++ b/backend/crates/gateway-plugin/sdk/docs/usage-data.md @@ -1,6 +1,8 @@ # 用量维护与计价 -`call::services::usage` 提供请求明细、聚合统计和 Token 费用试算。插件通过 `call.host.service::(input)` 调用，沿用公开服务的权限与调用期限 +[返回 SDK](../README.md) · [公开服务](capabilities.md#公开服务) + +`call::services::usage` 提供请求明细、聚合统计和 Token 费用试算。插件通过 `call.host.service::(input)` 调用，沿用公开服务的调用身份、期限与取消约束 ## 基础操作 diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index e764367af..9a121f4be 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -90,6 +90,17 @@ impl PgAccountGroupRepository { Ok(record) } + async fn committed_record(&self, id: &AccountGroupId) -> Option { + // 配置、版本和审计已经提交，展示查询失败不能把成功的变更变成错误返回。 + match self.required_record(id).await { + Ok(record) => Some(record), + Err(error) => { + tracing::warn!(group_id = %id.as_str(), error = %error, "committed account group projection is unavailable"); + None + } + } + } + async fn mutate( &self, audit: super::AdminAuditEvent, @@ -317,7 +328,7 @@ impl AccountGroupStore for PgAccountGroupRepository { Ok(AccountGroupMutation { config_revision: revision, id: id.clone(), - record: Some(self.required_record(&id).await?), + record: self.committed_record(&id).await, }) } @@ -363,7 +374,7 @@ impl AccountGroupStore for PgAccountGroupRepository { Ok(AccountGroupMutation { config_revision: revision, id: id.clone(), - record: Some(self.required_record(&id).await?), + record: self.committed_record(&id).await, }) } @@ -399,7 +410,7 @@ impl AccountGroupStore for PgAccountGroupRepository { Ok(AccountGroupMutation { config_revision: revision, id: id.clone(), - record: Some(self.required_record(&id).await?), + record: self.committed_record(&id).await, }) } diff --git a/backend/crates/gateway-store/src/postgres/backup.rs b/backend/crates/gateway-store/src/postgres/backup.rs index 8f4a0a83d..f1979de76 100644 --- a/backend/crates/gateway-store/src/postgres/backup.rs +++ b/backend/crates/gateway-store/src/postgres/backup.rs @@ -241,13 +241,27 @@ impl BackupRepository for PgBackupRepository { Ok(result.rows_affected() == 1) } - async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + async fn insert_backup_record( + &self, + seed: BackupRecordSeed, + context: &MutationContext, + ) -> AdminStoreResult { let mut transaction = self.pool.begin().await.map_err(|source| { store_unavailable("begin backup record insert").with_source(source) })?; - let result = async { + let result: AdminStoreResult<_> = async { let _ = lock_settings_in_transaction(&mut transaction).await?; - insert_queued_in_transaction(&mut transaction, &seed).await + let record = insert_queued_in_transaction(&mut transaction, &seed).await?; + let audit = crate::mutation_audit( + context, + MutationAuditOperation::BackupCreate, + &record.id, + vec!["status".to_owned()], + ); + append_audit_in_transaction(&mut transaction, audit, None) + .await + .map_err(map_admin_error)?; + Ok(record) } .await; match result { @@ -550,22 +564,56 @@ impl BackupRepository for PgBackupRepository { &self, id: &str, now: DateTime, + context: Option<&MutationContext>, ) -> AdminStoreResult> { - sqlx::query( - "update backup_records - set status = 'deleting', updated_at = $1 - where id = $2 and status in ('completed', 'failed') - returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, - attempt_count, error_code, error_message, started_at, completed_at, expires_at, - created_at, updated_at", - ) - .bind(now) - .bind(id) - .fetch_optional(&self.pool) - .await - .map_err(|source| store_unavailable("transition backup to deleting").with_source(source))? - .map(|row| backup_record_from_row(&row)) - .transpose() + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup deletion request").with_source(source) + })?; + let result: AdminStoreResult<_> = async { + let record = sqlx::query( + "update backup_records + set status = 'deleting', updated_at = $1 + where id = $2 and status in ('completed', 'failed') + returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at", + ) + .bind(now) + .bind(id) + .fetch_optional(&mut *transaction) + .await + .map_err(|source| store_unavailable("transition backup to deleting").with_source(source))? + .map(|row| backup_record_from_row(&row)) + .transpose()?; + if record.is_some() && let Some(context) = context { + let audit = crate::mutation_audit( + context, + MutationAuditOperation::BackupDelete, + id, + vec!["status".to_owned()], + ); + append_audit_in_transaction(&mut transaction, audit, None) + .await + .map_err(map_admin_error)?; + } + Ok(record) + } + .await; + match result { + Ok(record) => { + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup deletion request").with_source(source) + })?; + Ok(record) + } + Err(error) => { + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; + Err(error) + } + } } async fn delete_record(&self, id: &str) -> AdminStoreResult<()> { diff --git a/backend/crates/gateway-store/tests/bundle.rs b/backend/crates/gateway-store/tests/bundle.rs index 3a7481110..f52018309 100644 --- a/backend/crates/gateway-store/tests/bundle.rs +++ b/backend/crates/gateway-store/tests/bundle.rs @@ -11,6 +11,8 @@ use gateway_store::StoreConfig; use serde_json::json; use sqlx::postgres::PgPoolOptions; +use crate::support::config_connection; + #[tokio::test] async fn inspection_bundle_does_not_migrate_and_rejects_business_writes() { let (Some(database), Some(redis)) = ( @@ -228,12 +230,3 @@ async fn store_bundle_authenticates_postgres_password_with_percent_sequences() { .all(|state| *state == gateway_core::health::HealthState::Healthy) ); } - -fn config_connection(mut url: url::Url) -> serde_json::Value { - let password = percent_encoding::percent_decode_str(url.password().unwrap()) - .decode_utf8() - .expect("UTF-8 fixture password") - .into_owned(); - url.set_password(None).unwrap(); - json!({"url": url.as_str(), "password": password}) -} diff --git a/backend/crates/gateway-store/tests/postgres/account_groups.rs b/backend/crates/gateway-store/tests/postgres/account_groups.rs index dcfd7320d..92298b711 100644 --- a/backend/crates/gateway-store/tests/postgres/account_groups.rs +++ b/backend/crates/gateway-store/tests/postgres/account_groups.rs @@ -826,3 +826,88 @@ async fn image_generation_defaults_to_allowed_for_new_groups() { assert!(allowed); database.close().await; } + +#[tokio::test] +async fn committed_group_mutations_succeed_when_the_usage_query_budget_is_exhausted() { + let Some(database) = TestDatabase::create("account_group_projection_failure").await else { + return; + }; + let budget = + ObservabilityQueryBudget::try_new(1, Duration::from_millis(1), Duration::from_secs(30)) + .unwrap(); + let held_budget = budget.clone(); + let held_pool = database.pool.clone(); + let (entered, waiting) = tokio::sync::oneshot::channel(); + let (release, released) = tokio::sync::oneshot::channel(); + let holder = tokio::spawn(async move { + held_budget + .run( + &held_pool, + "hold projection budget", + |connection| async move { + entered.send(()).unwrap(); + released.await.unwrap(); + drop(connection); + Ok(()) + }, + ) + .await + }); + waiting.await.unwrap(); + let groups = PgAccountGroupRepository::new(database.pool.clone(), budget); + let id = AccountGroupId::new(MIXED_GROUP).unwrap(); + let before = current_revision(&database.pool).await; + let created = groups + .create_account_group( + NewAccountGroup { + id: id.clone(), + name: "Committed".to_owned(), + description: None, + color: AccountGroupColor::parse("#2563EBFF").unwrap(), + fast_mode: FastMode::Default, + allow_image_generation: true, + }, + &context("projection-create"), + ) + .await + .expect("create commits despite observation failure"); + assert!(created.record.is_none()); + let updated = groups + .update_account_group( + gateway_admin::model::account_groups::UpdateAccountGroup { + id: id.clone(), + name: "Updated".to_owned(), + description: None, + color: AccountGroupColor::parse("#2563EBFF").unwrap(), + fast_mode: None, + allow_image_generation: None, + }, + &context("projection-update"), + ) + .await + .expect("update commits despite observation failure"); + assert!(updated.record.is_none()); + let disabled = groups + .set_account_group_enabled( + gateway_admin::model::account_groups::SetAccountGroupEnabled { + id: id.clone(), + enabled: false, + }, + &context("projection-disable"), + ) + .await + .expect("disable commits despite observation failure"); + assert!(disabled.record.is_none()); + assert_eq!(current_revision(&database.pool).await, before + 3); + assert_eq!(audit_count(&database.pool).await, 3); + let persisted: (String, bool) = + sqlx::query_as("select name, enabled from account_groups where id = $1") + .bind(id.as_str()) + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(persisted, ("Updated".to_owned(), false)); + release.send(()).unwrap(); + holder.await.unwrap().unwrap(); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/backup.rs b/backend/crates/gateway-store/tests/postgres/backup.rs index 778a5fd92..f3dd77368 100644 --- a/backend/crates/gateway-store/tests/postgres/backup.rs +++ b/backend/crates/gateway-store/tests/postgres/backup.rs @@ -169,7 +169,10 @@ async fn backup_lifecycle_transitions_through_deleting_and_hard_delete() { let repository = PgBackupRepository::new(db.pool.clone()); let record = repository - .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .insert_backup_record( + seed(&"a".repeat(32), BackupTriggerKind::Manual, None), + &context(), + ) .await .expect("insert queued"); assert_eq!(record.status, BackupStatus::Queued); @@ -216,7 +219,7 @@ async fn backup_lifecycle_transitions_through_deleting_and_hard_delete() { assert!(completed.completed_at.is_some()); let deleting = repository - .transition_to_deleting(&record.id, Utc::now()) + .transition_to_deleting(&record.id, Utc::now(), Some(&context())) .await .expect("to deleting") .expect("transition matched"); @@ -245,11 +248,17 @@ async fn active_task_unique_index_blocks_second_queued() { let repository = PgBackupRepository::new(db.pool.clone()); repository - .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .insert_backup_record( + seed(&"a".repeat(32), BackupTriggerKind::Manual, None), + &context(), + ) .await .expect("first queued"); let error = repository - .insert_backup_record(seed(&"b".repeat(32), BackupTriggerKind::Manual, None)) + .insert_backup_record( + seed(&"b".repeat(32), BackupTriggerKind::Manual, None), + &context(), + ) .await .expect_err("second queued must conflict"); assert_eq!(error.kind(), AdminStoreErrorKind::Conflict); @@ -275,7 +284,10 @@ async fn active_task_unique_index_blocks_second_queued() { .await .expect("fail task"); repository - .insert_backup_record(seed(&"c".repeat(32), BackupTriggerKind::Manual, None)) + .insert_backup_record( + seed(&"c".repeat(32), BackupTriggerKind::Manual, None), + &context(), + ) .await .expect("third queued after terminal"); @@ -359,7 +371,10 @@ async fn storage_identity_is_locked_once_records_exist() { .expect("endpoint change allowed with no records"); repository - .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .insert_backup_record( + seed(&"a".repeat(32), BackupTriggerKind::Manual, None), + &context(), + ) .await .expect("insert queued"); @@ -505,3 +520,109 @@ async fn scheduled_insert_rejects_changed_schedule_and_stale_timezone_rebase() { assert_eq!(count, 0, "obsolete schedule never queues a task"); db.close().await; } + +#[tokio::test] +async fn manual_backup_mutations_roll_back_when_their_audit_cannot_be_written() { + let Some(db) = TestDatabase::create("backup_atomic_audit").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + let seed = seed(&"d".repeat(32), BackupTriggerKind::Manual, None); + let backup_id = seed.id.clone(); + sqlx::query("alter table admin_audit_events add constraint reject_backup_audit check (action not in ('backup.created', 'backup.delete_requested')) not valid") + .execute(&db.pool).await.expect("inject audit rejection"); + assert!( + repository + .insert_backup_record(seed.clone(), &context()) + .await + .is_err() + ); + assert!( + repository + .load_backup_record(&backup_id) + .await + .expect("load rolled back create") + .is_none() + ); + sqlx::query("alter table admin_audit_events drop constraint reject_backup_audit") + .execute(&db.pool) + .await + .expect("restore audit writes"); + repository + .insert_backup_record(seed, &context()) + .await + .expect("create with audit"); + repository + .claim_next_queued(Utc::now()) + .await + .expect("claim backup fixture") + .expect("queued backup is claimed"); + repository + .transition_status( + &backup_id, + transition(BackupStatus::Dumping, BackupStatus::Failed), + StatusTransitionUpdate { + error_code: Some("test_failure".to_owned()), + error_message: Some("synthetic fixture failure".to_owned()), + ..Default::default() + }, + Utc::now(), + ) + .await + .expect("finish backup fixture") + .expect("running backup becomes failed"); + sqlx::query("alter table admin_audit_events add constraint reject_backup_audit check (action <> 'backup.delete_requested') not valid") + .execute(&db.pool).await.expect("reject deletion audit"); + assert!( + repository + .transition_to_deleting(&backup_id, Utc::now(), Some(&context())) + .await + .is_err() + ); + assert_eq!( + repository + .load_backup_record(&backup_id) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Failed + ); + sqlx::query("alter table admin_audit_events drop constraint reject_backup_audit") + .execute(&db.pool) + .await + .expect("restore deletion audit"); + assert!( + repository + .transition_to_deleting(&backup_id, Utc::now(), Some(&context())) + .await + .unwrap() + .is_some() + ); + assert!( + repository + .transition_to_deleting(&backup_id, Utc::now(), Some(&context())) + .await + .unwrap() + .is_none() + ); + let events: Vec<(String, String, String)> = sqlx::query_as( + "select action, entity_ref, admin_request_id from admin_audit_events where entity_kind = 'backup' order by created_at, id" + ).fetch_all(&db.pool).await.expect("read committed audit events"); + assert_eq!( + events, + vec![ + ( + "backup.created".to_owned(), + backup_id.clone(), + context().request_id + ), + ( + "backup.delete_requested".to_owned(), + backup_id, + context().request_id + ), + ] + ); + db.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/pricing.rs b/backend/crates/gateway-store/tests/postgres/pricing.rs index af20fc386..4cf191ef2 100644 --- a/backend/crates/gateway-store/tests/postgres/pricing.rs +++ b/backend/crates/gateway-store/tests/postgres/pricing.rs @@ -20,14 +20,9 @@ async fn pricing_store(name: &str) -> Option<(TestDatabase, gateway_store::Store database_url .query_pairs_mut() .append_pair("options", &format!("-csearch_path={}", database.schema)); - let connection = |mut url: url::Url| { - let password = url.password().expect("test password").to_owned(); - url.set_password(None).unwrap(); - json!({"url":url.as_str(), "password":password}) - }; let mut config: gateway_store::StoreConfig = serde_json::from_value(json!({ - "database":connection(database_url), - "redis":connection(url::Url::parse(&redis_url).unwrap()), + "database":crate::support::config_connection(database_url), + "redis":crate::support::config_connection(url::Url::parse(&redis_url).unwrap()), })) .unwrap(); let runtime = tempfile::tempdir().unwrap(); diff --git a/backend/crates/gateway-store/tests/redis/worker_lease.rs b/backend/crates/gateway-store/tests/redis/worker_lease.rs index aa74d99d6..a44ff1388 100644 --- a/backend/crates/gateway-store/tests/redis/worker_lease.rs +++ b/backend/crates/gateway-store/tests/redis/worker_lease.rs @@ -127,19 +127,9 @@ impl TestDatabase { } fn store_config(database_url: &str, redis_url: &str) -> StoreConfig { - let (database_url, database_password) = split_connection_url(database_url); - let (redis_url, redis_password) = split_connection_url(redis_url); serde_json::from_value(serde_json::json!({ - "database": { "url": database_url, "password": database_password }, - "redis": { "url": redis_url, "password": redis_password }, + "database": crate::support::config_connection(url::Url::parse(database_url).unwrap()), + "redis": crate::support::config_connection(url::Url::parse(redis_url).unwrap()), })) .expect("test Store config") } - -fn split_connection_url(value: &str) -> (String, String) { - let mut url = url::Url::parse(value).expect("test connection URL"); - let password = url.password().expect("test connection password").to_owned(); - url.set_password(None) - .expect("connection URL supports credentials"); - (url.to_string(), password) -} diff --git a/backend/crates/gateway-store/tests/support.rs b/backend/crates/gateway-store/tests/support.rs index dd812ea28..6f5d83c6d 100644 --- a/backend/crates/gateway-store/tests/support.rs +++ b/backend/crates/gateway-store/tests/support.rs @@ -13,6 +13,16 @@ pub(crate) fn test_env(name: &str) -> Option { value } +/// Store 配置单独接收原始密码，不能把 URL 中的百分号编码当作密码内容。 +pub(crate) fn config_connection(mut url: url::Url) -> serde_json::Value { + let password = percent_encoding::percent_decode_str(url.password().expect("test password")) + .decode_utf8() + .expect("UTF-8 fixture password") + .into_owned(); + url.set_password(None).unwrap(); + serde_json::json!({"url": url.as_str(), "password": password}) +} + /// 按 is-ci 约定判定 CI：`CI` 存在且非 falsey（`0`/`false`/空） /// 开发者用 `CI=false` 显式声明「非 CI」时不应把跳过变成 panic fn running_in_ci() -> bool { diff --git a/backend/migrations/README.md b/backend/migrations/README.md index 7b341d9fc..9918992f3 100644 --- a/backend/migrations/README.md +++ b/backend/migrations/README.md @@ -3,7 +3,8 @@ sqlx 在服务启动、监听请求之前按编号执行迁移，并把文件 checksum 记入 `_sqlx_migrations`。 已应用迁移的字节与记录不一致时，服务会拒绝启动 -迁移按递增编号组织，完整清单以本目录 SQL 文件和 `.frozen-sha256` 为准 +迁移按递增编号组织，完整清单以本目录 SQL 文件和 `.frozen-sha256` 为准。 +业务状态与存储职责见[状态所有权](../../docs/architecture.md#8-状态所有权)，离线恢复见[部署文档](../../deploy/README.md#人工恢复数据库) ## 命名约定 @@ -33,14 +34,17 @@ sha256sum --check --strict .frozen-sha256 ## 本地测试库 -`gateway-store` 的 PG/Redis 集成测试需要以下环境变量，未设置时在本地 -静默跳过（CI 缺失则直接失败）： +`gateway-store` 的 PostgreSQL/Redis 集成测试使用以下环境变量；所需测试服务未配置时，本地相关测试跳过，CI 直接失败： ```bash export CPR_TEST_DATABASE_URL='postgres://:@127.0.0.1:5432/' export CPR_TEST_REDIS_URL='redis://:@127.0.0.1:6379' ``` +插件 Runtime 的真实子进程与持久化测试优先使用 `CPR_PLUGIN_TEST_DATABASE_URL` 和 +`CPR_PLUGIN_TEST_REDIS_URL`。设置插件数据库变量时必须同时提供对应 Redis 变量；未设置插件数据库变量时， +使用上述标准测试服务。两种配置遵守相同的密码与隔离要求；缺少服务时的跳过和 CI 失败规则也相同 + 测试自建随机 schema / key 前缀做隔离，但仍应使用专用开发或测试实例，不要连接生产库。 凭据从自己的部署配置或 CI Secret 中读取，不要粘贴未脱敏的 `docker inspect` 输出。 环境变量未设置导致的跳过不算数据库测试通过 diff --git a/deploy/README.md b/deploy/README.md index 251ee1e80..aca8644a6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -239,11 +239,11 @@ PostgreSQL/Redis 启动密码。日常校验使用 `config --quiet` 在“代理管理 → 新增代理”切换到“批量”，每行粘贴一个完整代理 URL、`host:port` 或 `host:port:用户名:密码`，最多 100 条。 完整 URL 使用各自协议，简写使用表单选择的协议；IPv6 主机使用方括号，简写中的密码可包含冒号。 名称按前缀和输入顺序生成，请求位置继承全局设置。勾选“添加后测试”时，只测试本次成功添加的记录。 -添加和测试结果合并在一个列表内，条目较多时滚动查看 +添加和测试结果合并展示，便于区分保存失败和连接测试失败 列表支持跨页选中，勾选代理后显示“批量测试”按钮，点击直接测试选中项。 -测试同时检测连通性和出口地区，地区列主行显示出口国旗、国家和城市，次行显示已配置的自定义请求位置。 -测试前读取并固定实际名单，同时执行两条，结果更新到列表。批量添加中的停止操作只阻止后续提交，已开始的请求仍会完成。 +测试同时检测连通性和出口地区；测试前固定实际名单，同时执行两条，结果更新到列表。 +批量添加中的停止操作只阻止后续提交，已开始的请求仍会完成。 离开代理页面停止后续提交，批量进度不在刷新后保留，已保存代理和测试结果仍可在列表查询 格式错误、重复或配置冲突不覆盖已有代理，也不阻止其他条目处理。 @@ -452,6 +452,8 @@ metrics_exporter = "none" WebSocket 会话完成，进程退出时强制断开剩余连接。已入队的观测写入与后台清理保留 `host.worker_shutdown_timeout_seconds`（默认 30 秒）的收尾预算；客户端在新进程就绪后重新连接 +管理端重启受理后，到进程退出期间，更新、回滚和再次重启均被拒绝；可继续查询更新状态 + Compose 的 `stop_grace_period` 为 75 秒，覆盖默认 30 秒后台收尾及进程清理余量。 调大后台关闭预算时，也必须同步预留容器停止宽限期，否则 Docker 会在宽限期结束时 SIGKILL @@ -646,8 +648,7 @@ exp 不进入正式版、alpha、beta 或 rc，普通实例也不能选择 Exp 其他通道的版本及发布说明，手动检查显示“当前没有可用更新”。检查失败单独报告，不能当作没有更新 alpha、beta、rc、exp 在 GitHub 标记为 Pre-release，不覆盖 GitHub Latest 或镜像 `latest`。 -允许连续升级的发行线从首次发布起遵守[迁移冻结规则](../backend/migrations/README.md#冻结规则)， -包括预发行到正式版的晋级；发版前验证对应升级路径，不能仅凭版本号认定数据库兼容 +升级前核对目标版本的数据库要求；可连续升级的发行线与约束见[迁移冻结规则](../backend/migrations/README.md#冻结规则) ### 管理端在线更新 @@ -763,8 +764,8 @@ OpenAI 主动额度重置卡及消费结果由上游持有，不属于本地备 - 备份暂存目录为 `host.runtime_data_dir/backup-staging`；Compose 默认对应 `/app/.runtime/data/backup-staging`，由 `.runtime/data` 卷持久化，权限 `0700`（仅 `cpr` 用户可读写）。归档流式压缩，只暂存一份压缩文件；部署卷至少预留一个最大业务归档的空间 -- S3/R2 存储、Cron 计划、保留策略与备份记录都保存在 PostgreSQL（`backup_settings` / - `backup_records`），备份记录行在删除成功后硬删除，操作历史进入 `admin_audit_events` +- S3/R2 存储、Cron 计划、保留策略与备份记录都保存在 PostgreSQL，备份记录在删除成功后移除。 + 人工创建和删除操作写入管理审计，自动保留清理不生成管理员操作审计 - 手工备份的 `expiresInDays` 在创建时生成独立 `expires_at`；计划备份按当前 `retentionDays` 生成 `expires_at`，并同时受 `retentionCount` 清理规则约束。到期只进入删除流程，不构成在线恢复点 diff --git a/docs/api.md b/docs/api.md index ab2fbe9ba..270341277 100644 --- a/docs/api.md +++ b/docs/api.md @@ -376,7 +376,7 @@ OAuth 账号默认 `prefer_websocket`，客户端使用 HTTP/SSE 时仍可能选 Codex OAuth backend 的候选上游为 WS 时，无 `previous_response_id` 的普通新链若规范化 `response.create` 达到 15 MiB，发送前选择 HTTP/SSE；HTTP 和 WS 入站均适用，下游交付协议不变。 该阈值为已观察到的上游消息大小边界预留余量，不是网关输入长度上限或 OpenAI 公布的统一限制； -API Key 账号不使用此大小策略。小请求、显式 warmup、未知外部 previous ID 和持久化续接保持原行为 +API Key 账号、小请求、显式 warmup、未知外部 previous ID 和持久化续接不因该大小策略改用 HTTP OAuth 的大 connection-local WS 续接，以及 HTTP `store=false` 成功后标记为 `ReplayRequired` 的 WS 增量，均在发送前返回 `status: 400 / previous_response_not_found`。客户端应清除旧 ID， @@ -444,8 +444,8 @@ Codex 专用目录中的 `context_window` 与 `max_context_window` 分别表示 #### OpenAI 透传 -OpenAI 路径保留客户端 Responses wire 语义：请求 body 的未知字段和字段顺序保持不变（受控模型 -映射除外），HTTP SSE 与 WebSocket 的上游业务事件除下述客户端错误兼容及响应头隔离外按原始字节转发， +OpenAI 路径保留客户端 Responses wire 语义：除[正文兼容](#正文兼容)、[身份归一](#请求头与身份)、模型映射及已配置的请求策略外， +请求 body 的未知字段和字段顺序保持不变。HTTP SSE 与 WebSocket 的上游业务事件除下述客户端错误兼容及响应头隔离外按原始字节转发， response ID 按 opaque 值处理而不假设 UUID 或固定长度；除客户端错误兼容与原生续写额度恢复外， OpenAI 上游错误 envelope 和允许下发的 opaque header 值也不由 canonical 观测结果重写。 `reasoning.effort` 的数字和未知扩展值保留给上游判断，本地观测不限制参数形态。 @@ -459,7 +459,7 @@ Images 固定使用 OpenAI Provider，不重建正文或映射模型字段；Pro Images 排除已知为 Free 套餐的 OAuth 账号；套餐缺失或未知时不推断为 Free，API Key 账号不套用该套餐规则。 其余账号仍需满足模型限制、额度等调度条件，实际生图权限由上游决定 -`/v1/alpha/search` 使用相同的 OpenAI Provider 原生端点边界：正文原样转发，模型不映射， +`/v1/alpha/search` 使用相同的 OpenAI Provider 原生端点边界：模型不映射，正文除会话 `id` 的账号归一及显式请求策略外原样转发， Provider 旁路读取请求中的 `model`，按[账号模型限制](#账号模型限制)选号。 `x-codex-turn-metadata` 在移除客户端账号身份并按当前 lease 重写 installation ID 后转发；上游账号 Authorization、Cookie、account ID、originator 和 User-Agent 均由代理安全重建。 @@ -480,6 +480,11 @@ xAI 的 `reasoning.effort` 接受 `none / minimal / low / medium / high / xhigh #### 容量拒绝与重试 +OpenAI HTTP 明确未发送的瞬时建连失败（含 TLS 握手 EOF）在单请求内共享最多 4 次 opening、30 秒预算。 +先在原账号上退避恢复；允许换号且至少剩余两次 opening 时，本次恢复最多使用一半剩余时间，并保留一次 opening。 +再次建连失败才按调度规则重选账号，选号不可用时使用原账号的剩余预算；固定账号或换号预算不足时仅在原账号恢复。 +证书、TLS 协议及配置错误不进入该恢复，取消与请求截止时间优先终止等待 + 客户端推理请求提供有效逻辑线程、轮次及 `request_kind=turn` 时，网关按 Client Key 隔离短期失败状态。 “同账号重试后返回错误”策略终态失败后，5 分钟内同轮次再次命中该策略时直接返回错误，不重复执行其最多 5 次重试； 再次失败刷新有效期，成功结束后解除限制。其他重试策略不受影响，缺少有效身份、状态过期或状态存储不可用时沿用原策略。 @@ -510,11 +515,16 @@ WebSocket 在交付有效输出前断线或超时，可按 `stream_max_retries` 次数限制不保证上游只执行一次，诊断保留原始发送状态。精确连接续接、显式终止拒绝、已观测的 外部副作用和下游提交仍限制恢复；取消与请求截止时间随时终止等待 -OpenAI 明确返回 `server_is_overloaded`、`slow_down` 或模型容量不足错误时，代理在允许安全重放且 -尚未交付输出的前提下，先做最多 3 次同账号重试，再通过现有调度换号。没有有效服务器建议时， -按 500ms 起步的指数退避等待，单次最多 8 秒；有效 `Retry-After` 优先，接受秒数、HTTP 日期和零延迟， -不受本地退避上限截断。流内失败事件使用相同解析规则，WS 转 HTTP 回退也遵守服务器建议； -等待受请求截止时间和取消约束，重试受请求总尝试次数约束。 +OpenAI 仅在允许安全重放且尚未交付不可重放输出时执行下表的同账号重试；次数指首次尝试之外的额外尝试： + +| 上游拒绝 | 同账号重试上限 | 没有有效 `Retry-After` 时的退避 | 耗尽动作 | +| --- | --- | --- | --- | +| `server_is_overloaded`、`slow_down` 或明确模型容量不足 | 3 次 | 500ms 起步的指数退避，单次最多 8 秒 | 按调度规则换号 | +| 普通 `503` 且 `error.code=server_error` | 5 次 | 500ms 起步的指数退避，基准最多 2 秒，每次追加 `[50ms, 250ms)` 等待 | 返回最后一次上游错误，不因此换号 | + +有效 `Retry-After` 优先，接受秒数、HTTP 日期和零延迟，不受本地退避上限截断，也不追加随机等待。 +流内失败事件使用相同解析规则，WS 转 HTTP 回退也遵守服务器建议；等待受请求截止时间和取消约束， +重试受[请求总尝试次数](#账号亲和与换号)约束。 已发送的容量失败计入[智能调度健康分](architecture.md#智能调度)，可用账号的会话亲和优先。 容量不足不触发 Provider 全局熔断，也不作为账号额度耗尽；启用账号自动冻结且达到阈值时写入临时冷却。 客户端错误兼容由 API 编码出口统一处理：最终交付的 `server_is_overloaded`、`slow_down` 错误码 @@ -616,8 +626,7 @@ call 与账号的绑定保存在网关进程内，未连接时一小时过期； 管理员和密钥登录共用 `/api/auth/*`。登录模式 `mode` 只用于选择凭据验证方式，不直接授予权限； 验证成功后，由后端写入身份和绑定 ID。浏览器通过 `cpr_session` HttpOnly Cookie 访问业务接口， -管理员另持有仅发送到 `/api/auth` 的 `cpr_refresh` HttpOnly Cookie。原始 Key 不进入 URL、Pinia 或浏览器存储。 -登录页的切换只改变本地表单，不改变 URL。 +管理员另持有仅发送到 `/api/auth` 的 `cpr_refresh` HttpOnly Cookie。原始 Key 不进入 URL 或浏览器持久化存储。 管理员进入管理端；Key 登录后进入 `/key-usage`，只读取当前会话绑定 Key 的数据 | 方法 | 路由 | 请求 | 说明 | @@ -657,9 +666,7 @@ call 与账号的绑定保存在网关进程内，未连接时一小时过期； 拒绝时使用 `42901` 和 `Retry-After` 认证错误共用 `40101`（会话失效）、`40102`（凭据错误）和 `40301`（权限不足）。 -前端首次读取状态未认证时尝试续期；业务请求在 `40101` 后统一尝试一次会话恢复，并发请求等待同一恢复结果； -恢复成功后最多重放一次，确认失效才退出。 -`40102` 和 `40301` 不触发恢复，网络或依赖故障不清除身份 +`40102` 和 `40301` 不触发会话恢复，网络或依赖故障不清除身份；浏览器的统一恢复与请求重放职责见[登录与查询身份](architecture.md#登录与查询身份) ### Key 用量与客户端配置 @@ -697,8 +704,7 @@ Token 明细、费用明细、用时/首字与状态。Token 和费用复用现 version 返回 `{ version, gitSha }`，不接受查询参数，不包含部署模式、更新状态或内部诊断 config 返回 `{ name, plaintextKey }`，仅读取服务端会话绑定的当前 Key，不接受任何查询参数。 -使用统计页在打开“密钥配置”弹窗时读取，用于复制 Codex 配置文件或导入 CCSwitch； -明文不进入用量轮询响应或浏览器持久化存储，关闭弹窗后清除页面中的配置状态 +这是显式读取敏感凭据的入口，明文不进入用量响应；客户端应仅在需要配置凭据时调用，并避免持久化到浏览器 ## 5. 账号 @@ -756,8 +762,8 @@ config 返回 `{ name, plaintextKey }`，仅读取服务端会话绑定的当前 查询需要管理员会话，账号不存在返回 404；`page` 从 1 开始，`pageSize` 为 1–100，偏移不能超过 `u32` 范围。结果是查询时的实时快照，按租约有效期排列，续期和请求完成可能改变跨页位置；接口禁止缓存，不提供历史记录 模型目录导出保留上游原生模型对象和能力字段，不包含账号凭据；不支持 Codex 原生目录的账号不能导出。 -管理端下载文件名为 `cpr-model-catalog-<套餐>-<账号名称>.json`，文件正文为 `catalog`，可用于 -Codex 的 `model_catalog_json` 配置。账号设置保存不等待上游模型目录刷新完成 +响应中的 `catalog` 可直接保存为 JSON 文件，用于 Codex 的 `model_catalog_json` 配置。 +账号设置保存不等待上游模型目录刷新完成 账号限流详情在 `quota` 中返回：`rateLimitReason` 为 `upstream_rate_limit`（上游临时限流）、 `capacity_freeze`（容量错误触发自动冻结）或 `null`。`recoveryProbeRequired` 表示解除冻结是否需要成功探测； @@ -826,7 +832,7 @@ Search 的选号限制不约束插件在选号后主动改写正文。 所有端点要求管理员身份。所有响应只返回去掉认证信息的 `endpoint`，不会返回完整 URL -| 方法 / Method | 路径 / Path | 请求 / Request | 结果 / Result | +| 方法 | 路径 | 请求 | 结果 | | --- | --- | --- | --- | | `GET` | `/api/admin/proxies` | `page`、`pageSize`（1-200）、`search`（名称） | `{ items, page }` | | `GET` | `/api/admin/proxies/accounts` | `proxyId`、`page`、`pageSize`（1-200）、`search`（账号名称或邮箱） | `{ items, page }` | @@ -875,8 +881,7 @@ Search 的选号限制不约束插件在选号后主动改写正文。 `detectLocation: true` 在测试连接时解析出口位置；结果 `location.status` 为 `notRequested`、 `detected`、`failed` 或 `conflict`。`detected` 携带 `location`，`failed` 携带安全错误说明； -位置查询失败不等同于代理连接失败。管理端的解析按钮把成功结果填入自定义位置表单，保存后生效； -失败时保留已有输入，不自动启用持续跟随。 +位置查询失败不等同于代理连接失败；检测结果不会自动开启持续跟随。 API 的 `autoLocation` 默认为 `false`；开启时使用已检测位置，测试连接会刷新检测结果。 `detectedLocation` 保存 `{ location, exitIpv4, exitIpv6, detectedAt }`，手动位置独立保留 @@ -1122,7 +1127,7 @@ OAuth start 使用： `email` 可缺失；不从导入文件的身份字段或附带的 ID token 回退补齐。验证失败不导入，接口区分提示 PAT 格式无效、被上游拒绝、验证服务不可用和身份响应无效，不回显令牌或原始上游响应。 PAT 不保存附带的 refresh token、ID token 或推测的过期时间，不参加 OAuth RT 刷新； - 失效后需取得新 PAT 再导入。普通 JWT 导入行为不变 + 失效后需取得新 PAT 再导入 - 首次 OAuth 保留回调 `state`、PKCE 与官方 token exchange，并持久化 `idToken`、`accessToken`、 `refreshToken`。刷新响应中的三个 token 字段均按官方语义独立轮换：返回新值时替换，省略时分别保留 现值。重新授权也保留这些回调保护，但只轮换目标账号的 token。回调地址只承载 `code`/`state`， @@ -1131,9 +1136,9 @@ OAuth start 使用： 观测完成才返回成功。观测失败只记录告警，不回滚已提交的账号；手工或后台 RT 刷新只更新 token， 不隐式等同于手工额度刷新，也不更新既有账号资料或 OAuth principal。xAI 导入与 OAuth complete 使用相同的提交后观察流程 -- OAuth pending flow 先取得带过期时间的独占 claim，只有账号事务提交成功后才消费。失败会释放 claim， - 但上游 authorization code 本身通常只能交换一次；已完成过 token exchange 时应重新创建 OAuth flow -- `GET /accounts/quota` 只读取最后一次落库快照；`POST /accounts/quota/refresh` 才访问上游。access token +- 同一 OAuth flow 不能并发完成；账号事务提交成功后保留可重试读取的完成结果。 + 失败会释放流程占用，但上游 authorization code 通常只能交换一次；已完成 token exchange 而未提交账号时应重新创建 OAuth flow +- `GET /api/admin/accounts/quota` 只读取最后一次落库快照；`POST /api/admin/accounts/quota/refresh` 才访问上游。access token 已过期时，额度刷新要求先走 credential 刷新或重新授权，不会拿过期 token 探测额度 - OpenAI 已耗尽账号首次可复核，在 `resetAt` 尚未到达或未知时按 30 分钟间隔兜底，也可发现 官方提前重置。到达最早未恢复窗口的 `resetAt` 后，先等待 10 秒传播余量，再在下一轮后台检查 @@ -1146,7 +1151,7 @@ OAuth start 使用： - 账号 `enabled` 控制是否参与请求调度，不控制 OAuth 凭据续期。OpenAI 与 xAI 的停用账号仍按各自刷新 策略维护 Token；刷新结果更新凭据状态，但不会启用调度。无 refresh token 或凭据已进入失效、无效、 封禁状态的账号不参加后台自动续期 -- `POST /accounts/recover` 对停用账号只将 `enabled` 改为 `true`，保留已有额度快照、凭据、错误和 Redis +- `POST /api/admin/accounts/recover` 对停用账号只将 `enabled` 改为 `true`，保留已有额度快照、凭据、错误和 Redis cooldown；启用后仍按这些事实投影状态，不会把已有错误或耗尽改成正常。对已启用账号则执行强制恢复： 清除 Redis cooldown 和已保存的额度/错误，恢复为可调度 credential。两条路径均不访问上游；强制恢复 不验证上游账号是否已经恢复，下一次真实请求仍可重新写入失败事实 @@ -1356,6 +1361,9 @@ HTTP 返回 `403` 和 `image_generation_disabled`，WebSocket 每个 `response.c | `POST` | `/api/admin/account-groups/disable` | `{ id }` | 禁用；已绑定 Key 保持受限，不回退到全部账号 | | `POST` | `/api/admin/account-groups/delete` | `{ id }` | 删除未被 Client Key 引用的组 | +写入返回 `{ id, record, configRevision }`。删除或已提交变更的展示数据暂不可用时，`record` 为 `null`； +客户端可重查列表获取展示数据 + 两种列表数据均为 `{ items, page, configRevision }`。完整列表 item 返回 `memberCount`、按 Provider 聚合的 `providerCounts` 和 `clientKeyCount`。查询分组成员使用账号列表的 `groupId` 筛选， 不提供独立的分组成员路由；账号的 Provider 不代表整个分组的 Provider。 @@ -1492,7 +1500,7 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly 位置检测返回 `status: detected` 与 `location`，双栈出口时区不一致时返回 `conflict`， 无法获取出口或位置时返回 `failed` 与 `message`。检测忽略进程代理环境变量，复用代理探测的超时与并发限制； -前端仅在成功时填入草稿，仍需保存设置才会生效 +检测结果须通过设置更新接口保存后才会生效 设置更新字段按用途分组如下： @@ -1519,7 +1527,7 @@ HTTP 返回 `429`，`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly `usageRetentionDays` 控制请求明细及其用量统计的保留天数，默认 31，支持 1～31 的整数天数。 `opsEventRetentionDays` 与 `auditRetentionDays` 分别控制独立运维事件和管理审计，默认 30、90，均须为正整数 -设置保存后由每小时运行的清理任务读取，按完成时间分批删除过期请求及关联事件，不删除运行中的请求。 +清理任务每轮读取已保存设置，按完成时间分批删除过期请求及关联事件，不删除运行中的请求；有界单轮结束后等待一分钟再检查。 缩短保留期后，已有过期数据逐批清理，清理结果不可恢复；Client Key 结算账本不受影响 统计在后台异步更新，清理期间可短暂包含已删除请求的统计贡献，投影完成后更新为剩余数据。 @@ -1581,9 +1589,9 @@ JSON 路径支持 `$`、`.成员`、`["带特殊字符的键"]`、`[索引]`、` 提交、分支、工作区标识和会话关联信息保持原值。正文中的工作目录及其他执行路径也保持原值， 供客户端工具使用，仍可能包含真实用户名和项目名。安装身份由账号隔离逻辑处理， UA、平台参数及自动 Git 上下文版本由 [OpenAI 上游客户端身份](#openai-上游客户端身份)管理。 -预设和自定义规则使用同一编辑器，步骤均可增删、修改和排序；调整预设不覆盖已保存的规则 +保存后的规则按其 `steps` 执行，不随预设定义变化 -一条规则内的所有步骤和承载副本共同提交或回滚。已有多条规则不会自动合并或被覆盖。目标为 `null` 时仍遵循类型校验与失败策略，路径键重名时失败，不合并不同工作区 +一条规则内的所有步骤和承载副本共同提交或回滚。目标为 `null` 时仍遵循类型校验与失败策略，路径键重名时失败，不合并不同工作区 正则使用 Rust `regex` 语法，不支持前后顾或模式反向引用；替换内容支持 `${1}`、`${name}` 和字面量 `$$`。 `replaceAll` 控制首个或全部匹配，另支持忽略大小写和多行锚点。最多 32 条规则，所有规则合计最多 32 个处理步骤；路径最多 16 层、1024 字节， @@ -2144,7 +2152,7 @@ OpenAI Responses 的 `provider.request.input` 与 `provider.request.outbound` `provider.response.usage` 从已解析的上游终态单独提取已知数值字段，区分缺失、无效和明确的零， 同时保存共享 parser 的解析结果；`request.usage` 保存最终提交持久化前的网关归集值。 `provider.response.parse_gap` 标记 JSON 或 canonical 投影失败，原始 wire 仍按原有规则透传。 -上述关键事实复用现有 trace JSON 随请求终态保存，不新增诊断表或列；旧记录不能补生成未采集的事实 +上述关键事实随请求终态保存；未采集的事实无法由导出补回 采集阶段仍按有界 trace 合同处理未知 JSON 键和值，未知事件名称保留摘要；导出不会补造被省略的原文。 下载端按 Provider 凭据、管理认证与传输头合同移除令牌、密钥、认证头、Cookie 和 URL 认证部分， @@ -2169,7 +2177,7 @@ OpenAI 与 xAI 的本地费用估算按实际发送给上游的请求模型（`u xAI 的 `cost_in_usd_ticks=0` 表示上游没有报告费用，不证明请求免费；正数才作为上游账单。 未报告时，只有已知价格且能覆盖本次费用的请求才使用本地估算，未知价格或无法覆盖的托管工具费用保持未知。 -动态 latest 与旧 beta/internal 别名不借用具体旧型号的内置价格；显式配置的模型价格仍可用于估算 +动态 latest 与 beta/internal 别名不借用具体型号的内置价格；显式配置的模型价格仍可用于估算 OpenAI Responses 用量记录的 `serviceTier` 与本地费用估算统一采用 Provider 最终发给上游的请求 `service_tier`，不使用响应档位覆盖或回退。例如发送 `priority`、响应回显 `default` 时，仍显示 @@ -2182,7 +2190,7 @@ Provider metadata 分别保留 `requestedServiceTier` 与 `upstreamServiceTier` 原始 `response.service_tier` 不变。用量中的 Fast 仅表示发送档位，不能证明上游实际加速，本地费用 估算也不能代替官方账单。档位与费用在请求记录生成时确定；查询不会回填历史档位或重算已存储费用 -本地计价使用[模型定价](#模型定价)的生效规则。缺少内置、同步及人工价格时不生成估价。新请求的本地 +本地计价使用[模型定价](#模型定价)的生效规则。缺少内置、同步及人工价格时不生成估价。本地 费用明细、有效单价、服务档位与自定义倍率随终态记录持久化，后续改价和清除覆盖不重算历史明细。 记录没有费用快照时，按内置规则核对总额后补充拆分，核对失败只显示原总额。 `billing.longContextBillingApplied` 表示已应用长上下文价格区间，与服务档位、自定义倍率独立； @@ -2233,6 +2241,8 @@ Provider metadata 分别保留 `requestedServiceTier` 与 `upstreamServiceTier` 服务端在重启锁内重新检查，目标或配置变化返回 `40901`，须重新检查并确认。确认后保留启用配置，重启时逐个尝试加载插件。 没有兼容性风险时可省略确认，不切换插件版本、接受其他制品或改写配置版本 +重启受理后，旧服务拒绝更新、回滚和重复重启，返回 `40901`；客户端等待新进程就绪后重新连接 + 状态响应的 `currentVersion` 表示已安装文件的版本，运行中的版本仍以 `/version` 为准。 `needRestart=true` 表示已验证的安装文件尚未在当前进程生效，此时应调用重启接口，不能重复发起更新或切换通道。 最近一次 `operation` 仅表示操作历史，成功记录不等于待重启，也不改变远端 `hasUpdate`。 @@ -2311,8 +2321,9 @@ URL 与 GitHub 的 `location.sha256` 在解析时均可省略，由服务端计 确认安装时，将解析返回的 `metadata.pluginId`、`metadata.version` 和 `metadata.sha256` 分别填入 `pluginId`、`version`、`location.sha256`，与原来的来源、凭据和代理选择一起提交 `artifacts/install`。 -这三个字段在远程安装时必填，确保实际安装内容与确认的包相同。清单 ID、版本、宿主兼容范围与平台 -必须通过校验；来源类型、地址或代理选择变化返回冲突，需要先显式修改更新来源。相同包及来源重复安装为幂等操作； +这三个字段在远程安装时必填，确保实际安装内容与确认的包相同。清单身份、版本、平台、摘要和包结构 +必须通过校验；宿主范围与能力版本的兼容提醒见[创建与更新](#创建与更新)。 +来源类型、地址或代理选择变化返回冲突，需要先显式修改更新来源。相同包及来源重复安装为幂等操作； 同一插件版本的平台产物不允许替换成不同内容。安装新版本保留已安装旧版本 ```json @@ -2387,7 +2398,7 @@ GitHub 的 `location` 使用 `kind: "github"`、`repository: "owner/repo"`、`ta 检查更新请求为 `{ "pluginId": "acme.example", "credentialIds": [] }`，返回 `binding` 和 `release`。 服务端使用已保存的来源、策略和代理，不接受临时替换仓库或代理；查询期间来源变更返回 `409`，不可访问或限流仍返回错误。 -`release` 仅是来源元数据，不表示插件包已通过身份、版本、平台、兼容范围或摘要校验；选择产物后仍须显式调用安装接口， +`release` 仅是来源元数据，不表示插件包已完成身份、版本、平台和摘要校验或兼容风险检查；选择产物后仍须显式调用安装接口， 安装成功也不会自动修改现有实例的固定制品。手动策略和不支持 Release 查询的来源返回明确错误，不伪装为没有更新 ### 12.1 运行实例 @@ -2407,7 +2418,7 @@ GitHub 的 `location` 使用 `kind: "github"`、`repository: "owner/repo"`、`ta #### 创建与更新 创建与更新的配置字段为 `name`、`artifactSha256`、`enabled`、`configuration` 和必填的 `bindings`， -`secrets` 可选。输入严格拒绝未知字段；`trustedProcess` 和 `grants` 不是输入字段。`artifactSha256` 必须指向 +`secrets` 可选。输入严格拒绝未知字段。`artifactSha256` 必须指向 已接受的制品。`configuration` 是匹配插件 schema 的 JSON 对象；敏感字段必须 放入 `secrets`，空对象清除全部值。省略时同版本编辑保留当前值；跨版本编辑优先保留目标版本快照的密钥，没有快照则保留当前值 @@ -2543,7 +2554,10 @@ CLI 插件也须在具体调用中选择当前 Key。管理员身份本身不提 管理 API 转交方法、相对路径、查询字符串、完整请求头、宿主请求关联与原始正文，保留 Cookie、认证头、 重复头与非 UTF-8 值。仅接受注册的方法、路径和内容类型；查询最多 8 KiB，正文最多 1 MiB 且受运行帧预算约束， GET/HEAD 不接受正文。响应状态码为 200–599，内容类型元数据须在注册列表内，204/304 不允许正文； -插件返回的完整响应头在 HTTP 语法校验后覆盖默认值。失败可能发生在副作用之后，不提供自动重试 +插件返回的完整响应头在 HTTP 语法校验后覆盖默认值。插件操作失败可能发生在副作用之后，不自动重试 + +插件原始响应与模型协议响应带宿主保留的 `X-CPR-Plugin-Response: 1`，插件不能覆盖此字段。 +宿主管理错误不带此标记；页面桥据此区分管理信封与插件正文，仅宿主 `40101` 触发[统一会话恢复](#统一登录与会话) 动态 API 与登录回调设置 `Cache-Control: no-store`。静态资源成功响应设置 `private, no-cache` 与内容 ETag， 浏览器可保存正文，但每次复用前必须重新校验管理员身份（公开资源除外）、实例版本和资源授权； diff --git a/docs/architecture.md b/docs/architecture.md index 17c732034..2522179d2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,7 +36,9 @@ flowchart TB Core --> Registry[Provider Registry] Admin --> Registry Registry --> Builtin[OpenAI / xAI] - Registry --> Plugins[Plugin Runtime / 插件进程] + Core -. 请求扩展 .-> Plugins[Plugin Runtime / 插件进程] + Admin -. 安装与发布 .-> Plugins + API -. 入口与管理扩展 .-> Plugins Builtin --> Upstream[上游服务] Plugins --> Upstream Core --> Store[gateway-store] @@ -225,7 +227,7 @@ flowchart LR 插件以与宿主相同的 OS 身份运行，安装者承担插件安全；宿主不提供插件安全沙箱或访问域隔离。 进程、RPC、下载、缓存与回调均有容量和期限限制；Host 提供受管资源，Runtime 负责协议、调用关联与资源生命周期。 -RPC 的调用数、队列与流窗口有界，完整载荷尚无独立的总驻留内存配额。 +RPC 的调用数、队列与流窗口有界，完整载荷没有独立的总驻留内存配额。 下载来源、凭据与代理按目标匹配，网络操作不占用数据库事务，提交前重新校验来源与代理版本 管理页面使用无同源权限的 sandbox iframe，通过固定目标的宿主桥访问已声明路由。 @@ -235,9 +237,6 @@ RPC 的调用数、队列与流窗口有界，完整载荷尚无独立的总驻 安装到使用见 [插件使用](plugins.md)，清单与能力合同见 [SDK](../backend/crates/gateway-plugin/sdk/README.md)， HTTP 字段见 [插件 API](api.md#12-插件管理)，更新与数据恢复见 [部署说明](../deploy/README.md#插件兼容与发行目录) -组合根的 workspace architecture tests 冻结成员清单、依赖 DAG、公开模块面、源码纪律以及生产/测试模块 -镜像关系；具体行为边界由各 crate 的集成测试维护 - ### 3.2 Rust 模块组织约定 后端采用目录模块的 `mod.rs` 风格；以下规则由组合根的 workspace architecture tests 扫描全部生产源码与 @@ -311,8 +310,8 @@ Store 将策略与设置 revision 原子持久化；编译结果随快照、计 每条规则跨承载副本原子提交；发送前拒绝携带 `NotSent` 和禁止重试标记。被规则改写的握手头参与 WS 池身份，防止复用旧值， 关闭策略不进入改写管线。字段与执行限制见[隐私策略 API](api.md#隐私策略) -Fast 限制由同一快照链路冻结：Client Key 所有绑定分组的开关取逻辑或； -禁用分组仍贡献限制，无分组 Key 不限制 Fast，不按所选账号的分组重新解释。 +Fast 策略由 Core 合并 Client Key 的全部绑定分组并随同一快照链路冻结，禁用分组也参与合并； +不按所选账号的分组重新解释。三态覆盖顺序与字段合同见[账号分组 API](api.md#6-账号分组)。 OpenAI Provider 在独立编码请求上、生成上游头与观测前统一应用顶层档位覆盖，HTTP、WS 与重试共用。 该策略不改变选号、会话亲和或共享原始请求；每个新 WS 请求重新取得当前策略。 WS 路由提示属于握手，连接复用时不重发；档位变化不重建连接或打断 `previous_response_id` 续写， @@ -336,7 +335,7 @@ Client Key 费用账本独立累计各次 attempt 的实际费用，不能因请 | 入口 | 职责 | | --- | --- | | `api/modules/` | 页面 HTTP 合同，模块名对应 `views/` 一级目录；复杂模块在同名目录内按功能拆分，业务类型由所属模块定义 | -| `api/request/` | 请求入口与插件链；负责会话恢复、重试、错误提示、错误归一化和原始响应解析，页面不重复解释响应信封 | +| `api/request/` | 请求入口与插件链；负责会话恢复、重试、错误提示、错误归一化和原始／流式响应处理，页面不重复解释响应信封 | | `stores/modules/`、`router/`、`plugins/` | 应用级登录与界面状态、路由和初始化；服务端数据的权威仍在后端 | | `views/<页面>/index.vue`、`views/<页面>/components/` | 页面组合与局部交互；复杂组件按职责拆成目录入口和相邻模块 | | `views/<页面>/composables/` | 页面查询、轮询、表单及操作生命周期 | @@ -345,8 +344,12 @@ Client Key 费用账本独立累计各次 attempt 的实际费用，不能因请 | `composables/` | 请求取消与过期结果隔离、分页、选择、异步动作、目录加载等共享机制 | | `utils/` | 按用途分开的数据解析、数字与时长格式化、位置校验、客户端配置、插件导航及颜色工具 | +JSON、原始字节与流式请求共用会话恢复和身份代际校验，身份变化后的迟到结果按取消处理。 +流式成功正文由消费方持有读取、期限与取消；交付前失败时，请求层终止底层传输 + 概览的数据请求和刷新由 `useDashboard` 持有，`views/dashboard/presenter.ts` 生成展示模型。 -Key 创建与编辑共用 `useApiKeyEditor` 和 `ApiKeyFormModal`，创建结果与密钥使用弹窗各自管理明文生命周期。 +Key 创建与编辑共用 `useApiKeyEditor` 和 `ApiKeyFormModal`，`useApiKeyUse` 持有创建结果与使用操作的明文， +关闭对应弹窗或页面销毁时清除。 用量表格的列、展示投影及健康时间线规则集中在 `components/usage/shared/`，管理页面与 Key 用量页共用展示， 各自的查询仍使用对应身份允许的 API @@ -375,7 +378,7 @@ sequenceDiagram E-->>A: 提交响应流 A-->>C: JSON / SSE / WebSocket E->>S: 幂等费用结算 - E->>S: 异步投递观测与计量 + E->>S: 异步投递请求观测 ``` 请求开始时冻结 `RuntimeSnapshot`、Client Key 的账号范围及账号模型政策、模型映射、Codex 客户端最低版本、Provider @@ -396,27 +399,22 @@ Client Key 鉴权完成后，API adapter 从有界请求头识别客户端，Cor - OpenAI 的自动恢复限于明确识别的账号、容量、`503 + error.code=server_error`、续接和传输场景； 其他上游拒绝保留响应与重试提示，交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围，明确的 `426` 仍允许 HTTP 回退 - `not_sent`、`sent`、`ambiguous` 是单调的上游发送边界；结果不明确时不能假定上游未收到请求 -- HTTP 明确未发送的瞬时建连失败（含 TLS 握手 EOF）共享最多 4 次 opening、30 秒的请求内预算； - 先在原账号、HTTP 传输上退避恢复，允许换号且至少剩余两次 opening 时，本次恢复最多使用一半剩余时间， - 留下时间和一次 opening；再次建连失败才排除原账号并以 HTTP 重选，选号不可用时回原账号使用剩余预算； +- OpenAI HTTP 明确未发送的瞬时建连失败共享请求级 opening 预算，先在原账号、HTTP 传输上退避恢复； + 允许换号且预算足够时，为重选保留时间和连接次数；再次建连失败才排除原账号并以 HTTP 重选，选号不可用时回原账号使用剩余预算； 外部指定账号或换号预算不足时保留同账号恢复，账号与容量拒绝仍按各自的安全重放规则判断换号； 证书、TLS 协议及配置错误不进入同账号建连恢复，取消和执行 deadline 优先终止等待 - downstream commit 记录不可撤回的首次交付；不可重放事件交付后禁止换号、重试和 fallback HTTP 保活可先提交响应头，Core 记录该首字节时间与实际状态，但首个业务批次仍独立提交； SSE 注释不构成业务输出或上游已发送的证明，也不改变重放资格 - 原生 Responses HTTP/SSE 与 WebSocket 即时交付正常事件，仅 Provider 明确标记的可替换前导保留重试资格；Core 单独维护重放屏障， - 仍检查错误分类、续接归属、发送状态、副作用及预算。中间件链不启用此前导例外 + Core 单独维护重放屏障，仅 Provider 明确标记的可替换前导保留重试资格，仍检查错误分类、续接归属、发送状态、副作用及预算 - API 在最终错误编码出口投影客户端恢复信号；该投影不修改 Provider 上游事实，也不改变 Core 的 重试与提交边界。具体错误合同见 [数据面接口](api.md#3-openai-数据面与模型目录) - Provider 为有界同账号退避显式选择耗尽动作：允许换号，或返回客户端错误；Core 在既有安全重放边界内执行，按账号维护请求内预算 - OpenAI 明确容量拒绝在同账号额外重试最多 3 次后允许按普通规则换号，普通 `503 + error.code=server_error` 额外重试最多 5 次，耗尽后返回最后一次上游错误，不因此换号 - 本地指数退避从 500ms 开始，允许换号策略的基准上限为 8s，不换号策略为 2s；不换号策略每次另加 `[50ms, 250ms)` 随机等待 - 有效的上游 `Retry-After` 优先于本地退避，不受本地上限截断，也不叠加随机等待；等待仍受取消和请求 deadline 约束 - 该退避消耗总路由预算，与 WS 传输恢复、OAuth 刷新及账号额度冷却分开 + 该退避消耗总路由预算，与 WS 传输恢复、OAuth 刷新及账号额度冷却分开；具体次数、退避与 `Retry-After` 规则见[容量拒绝与重试](api.md#容量拒绝与重试) - `RetryThenReturnError` 的终态失败按 Client Key、逻辑线程与轮次写入 Redis，后续同轮次请求命中时跳过该策略的退避重试，直接交回错误；其他恢复策略不受影响 API 从明确的客户端推理请求提取轮次，Core 不解析协议正文；预热、压缩、内部调用和探测不参与该状态 - 标记在失败终态交付前写入并刷新 5 分钟 TTL；成功终态交付前按请求入场时读取的版本原子清理，迟到成功不能删除较新失败 - Redis 操作最多等待 100ms，读取失败沿用原重试策略，更新失败保留原响应；TTL 到期或状态丢失后重新获得预算，不承诺精确识别客户端重试意图 + 标记在失败终态交付前写入并续期；成功终态交付前按请求入场时读取的版本原子清理，迟到成功不能删除较新失败 + Redis 操作使用独立有界等待，读取失败沿用原重试策略，更新失败保留原响应；TTL 到期或状态丢失后重新获得预算，不承诺精确识别客户端重试意图 - OpenAI Provider 与 WS 传输层不暂存前导等待后续事件，正常帧按顺序即时交付； Provider 只保留单向推进的重放边界，实际输出、终态或未知事件关闭重放窗口。 中间件链中的正常事件同样即时交付，但不保留可替换前导的重试资格。 @@ -425,7 +423,7 @@ Client Key 鉴权完成后，API adapter 从有界请求头识别客户端，Cor 取消、deadline、已观测外部副作用及下游交付边界 - 换号预算从运行设置冻结到请求计划，选中账号与上一 attempt 不同时计数；首次选号、同账号重试和 选号阶段过滤候选不消耗预算。跨 Provider 候选推进与 continuation 排除重放同样受限，预算耗尽后 - 以最后一个原始上游错误终态；总路由尝试仍最多 32 次 + 以最后一个原始上游错误终态；配置与总尝试上限见[账号亲和与换号](api.md#账号亲和与换号) - Provider 显式标记的终止拒绝禁止自动重试与传输回退，优先于未发送状态和安全重放证明 - 跨 Provider 只在账号范围和能力都允许，且请求尚未到达上游或已被证明可安全重放时发生 - 可恢复观测写入失败不能替换已经确定的客户端协议结果 @@ -434,7 +432,13 @@ Responses 按模型目录编译候选；全局模型映射是精确映射，未 Images 与 standalone Search 是 OpenAI Provider 自有端点：两者都不参与文本模型映射，只在 Client Key 的账号范围确实包含 OpenAI 账号时生成单一 OpenAI 候选。Images 的图片模型由 Provider 从正文旁路 提取，用于账号模型政策检查，并在中间件改写后复验当前租约账号；不要求图片模型进入文本目录。 -Search body 中的模型及其他字段保持原始 bytes 并由上游解释 +Search 的请求身份与正文透传规则见 [OpenAI 透传](api.md#openai-透传) + +WebSocket 的根执行携带不解释协议的连接控制端口，Provider 在请求写入上游后绑定原 socket。 +API 在响应期间发送控制帧，终态后还可读取原连接事件；空闲读取必须在下一轮执行前取消， +与 Provider 正文消费共用单一有界接收通道。连接 owner 持有控制目标，复用时替换，销毁时释放； +控制句柄使用弱引用，不延长连接生命周期，也不经过全局响应查询、新选号或独立 attempt。 +API 暂存的下一轮请求和连接泵接收队列共用容量名额，控制帧处理不取消中间件正在读取响应体的 future ## 5. Provider 与协议边界 @@ -450,9 +454,8 @@ API Key 默认 HTTP/SSE，可选 WS 优先；选号先验证传输资格，WS po OAuth 与 API Key 共用业务请求、响应和能力透传链路，差异限定在上游地址、认证、传输配置及明确的上游请求合同适配。 OpenAI 模型目录用于发现，不因目录缺项拒绝请求；管理员配置的模型权限仍由 Core 与选号链路执行 -- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序；SSE、WebSocket、Images 与 standalone - Search 的业务正文按原始字节转发，Search 请求的会话 `id` 遵循账号归一规则， - 原生续写额度恢复遵循下述 continuation 例外。 +- OpenAI 是透明边界。正文兼容、身份归一和显式请求策略之外的业务字段保留原始语义； + 业务响应按原始字节转发，客户端错误兼容与原生续写额度恢复按明确例外处理，见 [OpenAI 透传](api.md#openai-透传)。 `transport/correlation` 在出站副本上统一映射账号关联标识并剥离客户端 trace， 内部选号、亲和、连接池与本地诊断继续读取原始身份；具体字段边界见 [API 协议](api.md#请求头与身份) canonical facts 从同一数据旁路提取，用于路由、恢复判断、观测和计费。 @@ -779,11 +782,6 @@ Admin 的手动重置复用同一账本与 Key 行锁，在一个事务中清零 - SSE 持有并发名额直到终态；WebSocket 每个 `response.create` 独立准入，空闲连接不占名额。 Core 在每次请求开始时重新鉴权并冻结当前策略，既有连接也应用已发布的限额与授权范围变更。 换号和内部重试复用同一名额，Key 与账号并发上限同时生效；预算拒绝或启动失败释放名额 -- Core 的根执行携带不解释协议的连接控制端口，Provider 在请求写入上游后绑定原 socket。 - API 在响应期间发送控制帧，终态后还可读取原连接事件；空闲读取必须在下一轮执行前取消， - 与 Provider 正文消费共用单一有界接收通道。连接 owner 持有控制目标，复用时替换，销毁时释放； - 控制句柄使用弱引用，不延长连接生命周期，也不经过全局响应查询、新选号或独立 attempt。 - API 暂存的下一轮请求和连接泵接收队列共用容量名额，控制帧处理不取消中间件正在读取响应体的 future 完成、失败、取消和断连使用同一结算路径，在结束时以网关请求 ID 幂等累计已取得费用。 执行会话持有结算与准入释放的同一个清理 future；协议等待被取消后，后续驱动或宿主 detached finalizer @@ -801,9 +799,9 @@ PostgreSQL 不可用时拒绝所有新的计费请求，Redis 继续管理并发 Admin 定义明细、聚合维护与纯计价合同，Provider 复用原生 Token 计价公式，Runtime 传递可信调用身份。插件负责外部格式、来源映射、去重、调整历史、目标选择和跨调用流程 -明细直接维护 `model_requests` 的终态记录，沿用现有统计失效触发器；计价快照在维护后清空，Key 结算账本保持独立。聚合贡献复用 `request_analytics_1h` 的 `collection`、维度、计数和费用结构，`usage_contribution` 集合保存调用方输入，其他集合仍为请求投影。该集合独立于请求投影，桶重建和版本重建保留该集合，查询在同一快照中按指定维度和归属时间合并贡献；聚合的删除由调用方管理 +明细直接维护 `model_requests` 的终态记录，通过统计失效触发器更新投影；计价快照在维护后清空，Key 结算账本保持独立。聚合贡献使用 `request_analytics_1h` 的 `usage_contribution` 集合保存调用方输入，其他集合为请求投影。桶重建和版本重建保留贡献集合，查询在同一快照中按指定维度和归属时间合并贡献；聚合的删除由调用方管理 -Store 在单次事务内完成有限批量读写和普通审计，复用观测连接与执行预算，不增加数据库表或字段。原生统计结果仍遵守现有数值类型，越界返回错误，不自动修正插件数据。调用合同与时间归属见 [SDK 用量维护与计价](../backend/crates/gateway-plugin/sdk/docs/usage-data.md) +Store 在单次事务内完成有限批量读写和普通审计，复用观测连接与执行预算。原生统计结果遵守各自数值类型，越界返回错误，不自动修正插件数据。调用合同与时间归属见 [SDK 用量维护与计价](../backend/crates/gateway-plugin/sdk/docs/usage-data.md) ### 模型价格与费用快照 @@ -839,6 +837,10 @@ HTTP validation 不构成所有写入的统一前置条件。运行设置整体替换必须携带读取版本，Store 在同一事务比较后才提交， 防止覆盖并发修改；插件、代理等资源使用各自的 `revision` / `expectedRevision`，不能与全局版本混用 +账号组的展示投影与配置提交结果分离；提交后的展示查询失败不阻止已提交版本发布，响应可不附带展示记录 + +人工备份创建与删除不推进配置 revision，Store 在同一事务提交任务状态与审计，Worker 只接手已提交任务 + 运行设置的共同管理值由 Admin 定义，读取结果与替换命令显式组合这些值，各自持有版本和 Provider 画像的读写语义。 Core 只持有请求所需设置与策略约束；Store 负责 SQL 解码、密钥隔离和事务，API 负责 wire 类型与展示。 HTTP 与插件的平铺合同独立于宿主内部组合，SDK 从宿主声明生成，不手工维护第二份字段来源 @@ -875,8 +877,7 @@ PostgreSQL 周期对账才是正确性基础 容量冻结与上游限流共用冷却投影，探测恢复与持久化并发调整由 Admin 负责，详见[Worker 与账号冻结](#worker-与账号冻结) -PostgreSQL schema 由迁移目录按编号管理。已应用迁移按字节冻结，后续 schema 变化只能新增编号迁移， -详见 [迁移规则](../backend/migrations/README.md) +PostgreSQL schema 由迁移目录按编号管理；冻结、升级与校验规则统一维护在[迁移文档](../backend/migrations/README.md) ## 9. Credential、额度与主动重置 @@ -999,8 +1000,12 @@ observation、调用 metadata；最终失败的关联头不与 opening 身份混 但在首次合法冷流建立前发生的无可用账号、准备失败、超时或取消，复用同一请求表补写失败终态与 诊断快照；没有实际 attempt 时计数为零，账号与传输保持空值。Provider 的准备过程不得发送本次请求的 上游握手或业务载荷，也不得启动独立发送任务；已合法登记冷流后，即使没有首事件也保留真实 attempt。 +这一冷流约束同样覆盖 attempt 中间件的响应读取，Core 校验元数据并登记 attempt 后才允许开始读取。 不能因首写失败或缺少 metadata 而伪造零尝试、账号、传输或 `not_sent` +Provider 流返回的错误、随附事件与重试决策由 Core 会话持有；取消单次等待只暂停该错误的处理， +不能因等待取消而重复读取上游或重建 attempt + 终态观测写入由会话持有；事件等待被取消后，后续 poll 或 detached finalizer 继续同一个 future， 保留原结果及完成时间，不重复创建请求。观测写入失败仍为 best-effort，不阻断费用结算与准入释放。 鉴权、解析、路由或准入等尚未进入执行会话的入口拒绝不纳入此请求记录边界；异步投影也仍可能延迟或 @@ -1019,7 +1024,7 @@ Provider metadata 的 `upstreamServiceTier`，不改写客户端收到的响应 Worker 由各 Bundle 贡献、由 Host 统一监督： - Store：过期请求恢复和 PostgreSQL/Redis 观测队列 -- Host：历史保留任务，持有每轮结束后一分钟的调度间隔、单轮行数/批数/时长预算、取消和日志；Admin 校验保留窗口，Store 执行有界批量删除 +- Host：历史保留任务，持有调度、单轮行数/批数/时长预算、取消和日志；Admin 校验[保留窗口](api.md#历史保留)，Store 执行有界批量删除 - Core：`runtime` owner 的 RuntimeSnapshot 周期对账和 Redis change 订阅 - Admin：S3/R2 备份 daemon，负责调度、执行、删除收敛与保留清理； 以及账号冻结恢复 worker（容量熔断的自适应并发下调与到期探测解冻） @@ -1097,6 +1102,7 @@ Compose 的 `stop_grace_period` 必须覆盖后台关闭预算并留出进程清 在线更新校验 Release host、大小、SHA-256、归档路径和发行身份，并只允许同一大版本内更新。 下载阶段不以插件合同不兼容阻止安装。重启前由 Admin 检查已安装发行与启用插件，返回绑定发行摘要和配置版本的确认快照。 Host 在重启操作锁内重新检查，Admin 复核风险确认与配置版本，不改写插件配置；重启后 Runtime 逐实例加载并隔离失败。 +重启受理后保持排他状态与跨进程文件锁，直到旧服务释放；延迟触发关闭仅用于送达受理响应。 更新与回滚在文件交换前后复核同一插件配置版本，失败或取消时恢复二进制、Web 与官方插件目录 Host 在受理时持久化任务并转交后台执行，任务持有操作锁与终态写入责任，不依赖 HTTP 请求的生命周期。 @@ -1142,6 +1148,9 @@ PostgreSQL 备份恢复属于人工维护操作。当前没有部署级维护模 ### 验证命令 +组合根的 workspace architecture tests 检查成员清单、依赖 DAG、公开模块面、源码纪律以及生产/测试模块镜像关系； +具体行为边界由各 crate 集成测试维护 + 后端验证从 `backend/Cargo.toml` 执行，仓库根目录没有 Cargo manifest： ```bash @@ -1150,18 +1159,10 @@ RUST_MIN_STACK=16777216 cargo +1.99.1 clippy --manifest-path backend/Cargo.toml RUST_MIN_STACK=16777216 cargo +1.99.1 test --manifest-path backend/Cargo.toml --test main --locked ``` -线程栈设置与当前 CI 一致。PostgreSQL/Redis 集成测试需按 -[迁移文档](../backend/migrations/README.md#本地测试库) 配置专用测试库；未设置环境变量时，本地相关测试会跳过。 +线程栈设置与当前 CI 一致。Store 与插件 Runtime 的 PostgreSQL/Redis 集成测试需按 +[本地测试库](../backend/migrations/README.md#本地测试库) 配置专用服务；未设置环境变量时，本地相关测试会跳过。 其他检查与界面验证按 [贡献与审查](../CONTRIBUTING.md#验证) 执行 -插件 Runtime 的真实子进程与持久化测试使用 `CPR_PLUGIN_TEST_DATABASE_URL` 和 -`CPR_PLUGIN_TEST_REDIS_URL` 指向专用实例；未提供插件专用变量时复用 `CPR_TEST_DATABASE_URL` 与 `CPR_TEST_REDIS_URL`。 -密码及隔离要求与上述 Store 测试一致，CI 缺少服务配置时直接失败。 - -测试归档缓存在 Cargo 测试临时目录的 `plugin-packages-v1/`，按含 worker 摘要的清单跨进程复用； -每项测试独立校验、解包并创建会话、子进程与 Store，缓存不承载可变运行状态 - -独立插件包的构建、安装与功能验证说明位于 `codex-proxy-plugins` 仓库的 `examples/workbench/README.md`。 -功能测试与性能、隔离和平台实测分别记录，不相互替代 +独立插件的本地构建与验证入口见[开发指南](development.md) 文档更新条件、内容归属与行文要求见[文档职责](../CONTRIBUTING.md#文档职责与更新条件) diff --git a/docs/development.md b/docs/development.md index a4166eb90..8d671d66a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -135,6 +135,9 @@ pnpm --dir modules/plugins/examples/workbench/frontend build:source ) ``` +独立插件的构建、打包和安装后功能验证见 [Workbench 示例](../modules/plugins/examples/workbench/README.md)。 +宿主插件集成测试的数据库与 Redis 配置见[本地测试库](../backend/migrations/README.md#本地测试库) + ## 编辑与提交 直接打开宿主目录即可开发，分别在对应仓库目录操作 Git diff --git a/docs/plugins.md b/docs/plugins.md index 1c6d76c33..2c45ef649 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -56,9 +56,8 @@ flowchart LR “事件观察”分别设置“请求完成与用量”和“上游 WebSocket 事件”；关闭其中一类不影响另一类。 观察用于接收事实；请求、响应或消息改写使用对应的中间件 -范围数组为空表示不限制该维度。自动创建的默认绑定不预先绑定 Client Key；普通网关请求仍以实际请求的 Key 和冻结策略 -执行。管理页面和命令行不保存功能 binding，页面调用模型时由页面每次明确选择一个 Client Key ID，宿主按当前 Key 状态、 -范围、预算、账本和普通请求链执行，页面桥使用 Key ID +范围数组为空表示不限制该维度。自动创建的默认绑定不预先绑定 Client Key；普通网关请求仍使用实际请求的 Key。 +管理页面和命令行调用模型时另行选择 Client Key，沿用该 Key 的启用状态、范围和预算，计入普通请求用量 `frontend_authentication` 不会随安装自动接管入口。启用它时必须明确配置外部 principal 到既有 Client Key 的映射； 已停用或删除的 Key 不会因此恢复访问 @@ -71,7 +70,7 @@ flowchart LR 删除插件配置也不自动删除资源。需要撤销 Key 访问时，在 Key 管理中停用或删除相应 Key “已启用”表示配置已经发布，不是外部服务的健康证明。“等待生效”表示候选仍在准备；“需要处理”会显示发布或运行失败原因。 -并发编辑冲突时应重新加载当前 revision 后确认，不能覆盖另一窗口的修改 +并发编辑冲突时应重新加载最新配置后确认，不能覆盖另一窗口的修改 ## 使用 @@ -124,13 +123,8 @@ flowchart LR 能力响应无法解码只使当前调用失败；上游 429、用户请求错误和插件主动拒绝请求等业务结果不会直接停止插件。其他插件、原生转发和管理端继续运行， 不因单个插件故障撤销整个宿主的运行配置。命中故障插件绑定的请求仍遵循既有拒绝或委托策略，入口认证插件失败不会绕过认证 -宿主每个插件会话默认允许 64 个执行中的 RPC 调用、64 个初次排队调用和 256 个总在途上下文。 -等待 `next` 时释放执行名额，返回后重新取得名额并继续原调用；已返回响应头的流继续占用在途预算，不占首结果执行名额。 -初次排队和恢复等待各限 30 秒，不扣插件自身默认上限 120 秒的处理预算。达到容量或等待上限时，中间件明确返回过载错误，HTTP 响应为 503（`middleware_capacity_unavailable`），不跳过插件、不重放请求。 -这些数值是有界资源预算的初值，实际容量需结合插件耗时、正文和流的资源占用评估 - -普通宿主回调默认上限 64，`next` 使用总在途上限对应的独立回调预算。实际容量由宿主预算与插件 SDK 会话配置共同限制； -调整插件侧容量需由作者重新构建插件包 +插件中间件达到调用容量或排队上限时，HTTP 响应为 503（`middleware_capacity_unavailable`），不会跳过插件或重放请求。 +应结合插件耗时和并发负载排查；会话预算与作者侧配置见 [SDK 并发与期限](../backend/crates/gateway-plugin/sdk/README.md#并发与期限) ### 停用与删除 diff --git a/docs/theme.md b/docs/theme.md index a59460832..048fdff7e 100644 --- a/docs/theme.md +++ b/docs/theme.md @@ -121,8 +121,8 @@ type ThemeMode = 'system' | 'light' | 'dark' type ThemeColorId = 'relay-blue' | 'deep-teal' | 'signal-violet' | 'graphite' | 'custom' interface ThemeCustomization { - seed?: Partial - component?: Partial + seed?: ThemeSeedOverrides + component?: ThemeComponentOverrides tokenOverrides?: Partial } ``` @@ -134,7 +134,8 @@ interface ThemeCustomization { | 中性色 | `colorTextBase`、`colorBgBase` | | 尺寸 | `fontSize`、`sizeUnit`、`sizeStep`、`controlHeight` | | 风格 | `borderRadius`、`shadowStrength` | -| 组件尺寸 | `tableRowHeight`、`cardBorderRadius` | + +表格行高与卡片圆角通过 `customization.component` 的 `tableRowHeight`、`cardBorderRadius` 配置，属于组件尺寸覆盖 持久化键为 `codex-proxy-rs-theme`。Store 不保存完整色板或 CSS Variables，损坏的模式、颜色与自定义值会在 初始化时规范化并回退到默认配置 @@ -234,8 +235,8 @@ Token 直接覆盖时不会自动重算同组件的其他状态；需要保持 | 角色 | 状态与可读性约束 | | --- | --- | | 主按钮 | 白字背景至少达到 4.5:1；Hover / Active 加深，不反向改写 Primary Seed | -| 输入控件 | Input、Select、Textarea、NumberInput 共用 Token；常态无边，Hover 与 Focus 保持相同背景和外圈 | -| 次级操作 | 普通按钮与图标按钮各用自己的 Secondary Token；透明填充与宿主表面合成，`ghost` 无常态底色 | +| 输入控件 | Input、Select、Textarea、InputNumber 共用 Token；常态无边，Hover 与 Focus 保持相同背景和外圈 | +| 次级操作 | 普通按钮与图标按钮各用自己的 Secondary Token；透明填充与宿主表面合成，`text` 与 `link` 无常态底色 | | 表格 | 表头、斑马纹、Hover 独立派生；固定列使用不透明合成背景，选中行保留 Primary 语义 | | 错误与焦点 | Hover / Focus 不覆盖错误反馈；焦点外圈不受装饰性阴影强度影响 | | 浮层头部 | 浅色取 Secondary Fill，深色取 Tertiary Fill；箭头与头部使用同一 Token，不借用表格背景 | @@ -383,7 +384,7 @@ Store 在其他入口发生变化时更新保存基线；没有未保存修改 共享包的 `styles/tailwind.css` 是唯一 `@theme inline` 注册表，只注册 Tailwind 名称，不保存主题值。注册表按基础、排版、颜色、圆角、阴影、间距与尺寸排序； 颜色再按基元、表面、主色与链接、语义、预设、数据、组件分组。 Preset 家族按字母排序，每个家族固定使用 `container → container-strong → border → solid → text → on-container`。主题值由 -`initializeTheme()` 在 Vue 挂载前动态生成并提交，不增加 `theme:generate`、`theme:check` 或构建期快照 +`initializeTheme()` 在 Vue 挂载前动态生成并提交，不维护构建期快照 全局元素基线统一放进共享包 `styles/base.css` 的 `@layer base`，确保组件 utility 可以按 Tailwind 层级正常覆盖；可复用的 原生滚动条声明使用 Tailwind CSS 4 `@utility`。组件内能等价表达的简单 SVG、渐变、原生外观与伪元素优先使用 diff --git a/frontend/src/api/modules/keys.ts b/frontend/src/api/modules/keys.ts index 41635e99a..cdd8fb7a4 100644 --- a/frontend/src/api/modules/keys.ts +++ b/frontend/src/api/modules/keys.ts @@ -121,11 +121,12 @@ export function updateApiKey(data: ApiKeyUpdateParam) { }) } -export function revealApiKey(data: ApiKeyIdParam) { +export function revealApiKey(data: ApiKeyIdParam, options: RequestOptions = {}) { return request({ url: '/api/admin/client-keys/reveal', method: 'GET', params: data, + ...options, }) } diff --git a/frontend/src/api/modules/plugins/extensions.ts b/frontend/src/api/modules/plugins/extensions.ts index 970de4c11..e6aaa52e8 100644 --- a/frontend/src/api/modules/plugins/extensions.ts +++ b/frontend/src/api/modules/plugins/extensions.ts @@ -1,11 +1,11 @@ import type { RequestConfig, RequestOptions } from '../../request' -import { API_BASE_URL } from '../../constants' -import request, { requestRaw } from '../../request' +import request, { requestRaw, requestStream } from '../../request' // 管理路由可能执行长任务，HTTP 与页面桥使用同一等待上限。 export const PLUGIN_MANAGEMENT_TIMEOUT_MS = 180_000 const MAXIMUM_MANAGEMENT_BODY_BYTES = 1024 * 1024 +const PLUGIN_RESPONSE_HEADER = 'x-cpr-plugin-response' export interface PluginManagementTarget { instanceId: string @@ -110,18 +110,21 @@ export function callPluginModelResponses( signal: AbortSignal, ) { const query = new URLSearchParams({ clientKeyId }) - return fetch(`${API_BASE_URL}${pluginManagementTargetPath(target)}/models/responses?${query}`, { + return requestStream({ + url: `${pluginManagementTargetPath(target)}/models/responses?${query}`, method: 'POST', - body, - cache: 'no-store', - credentials: 'same-origin', + data: body, + timeout: 0, + silent: true, headers: { 'accept': 'application/json, text/event-stream', 'Content-Type': 'application/json', }, - redirect: 'error', - referrerPolicy: 'no-referrer', + fetchOptions: { cache: 'no-store', redirect: 'error', referrerPolicy: 'no-referrer' }, signal, + }, { + maximumBytes: MAXIMUM_MANAGEMENT_BODY_BYTES, + accept: ({ header }) => header(PLUGIN_RESPONSE_HEADER) === '1', }) } @@ -157,10 +160,6 @@ function encodePluginPath(path: string) { function requestPluginManagementRaw(config: RequestConfig) { return requestRaw(config, { maximumBytes: MAXIMUM_MANAGEMENT_BODY_BYTES, - accept: ({ header }) => { - const policy = header('content-security-policy') - return policy?.split(';').some(directive => directive.trim() === 'sandbox allow-scripts') === true - && header('x-content-type-options')?.toLowerCase() === 'nosniff' - }, + accept: ({ header }) => header(PLUGIN_RESPONSE_HEADER) === '1', }) } diff --git a/frontend/src/api/request/index.ts b/frontend/src/api/request/index.ts index a97ab1549..654ca7148 100644 --- a/frontend/src/api/request/index.ts +++ b/frontend/src/api/request/index.ts @@ -35,8 +35,8 @@ const http = axios.create({ // 从外到内：提示、会话恢复、重试、错误归一化、原始响应处理、实际传输 const plugins: RequestPlugin[] = [feedbackPlugin, sessionPlugin, retryPlugin, errorPlugin, rawPlugin] -function execute(config: RequestConfig, raw?: RawResponseOptions) { - const context: RequestContext = { config, raw, isCurrent: () => !config.signal?.aborted } +function execute(config: RequestConfig, raw?: RawResponseOptions, stream = false) { + const context: RequestContext = { config, raw, stream, isCurrent: () => !config.signal?.aborted } const transport: RequestNext = () => http.request(context.config) return plugins.reduceRight((next, plugin) => () => plugin(context, next), transport)() } @@ -53,3 +53,17 @@ export async function requestRaw(config: RequestConfig, options: RawResponseOpti const { data } = await execute({ retry: false, ...config }, options) return data as RawResponse } + +export async function requestStream(config: RequestConfig, options: RawResponseOptions): Promise { + // 成功正文保持流式交付；宿主错误仍由同一管线识别和恢复，禁止网络错误重放写操作 + const controller = new AbortController() + const signal = config.signal ? AbortSignal.any([config.signal, controller.signal]) : controller.signal + try { + const { data } = await execute({ ...config, adapter: 'fetch', retry: false, signal }, options, true) + return data as Response + } + catch (error) { + controller.abort() + throw error + } +} diff --git a/frontend/src/api/request/plugins/index.ts b/frontend/src/api/request/plugins/index.ts index 5a9201630..ed61afa45 100644 --- a/frontend/src/api/request/plugins/index.ts +++ b/frontend/src/api/request/plugins/index.ts @@ -5,6 +5,7 @@ import type { RawResponseOptions } from './raw' export interface RequestContext { config: RequestConfig raw?: RawResponseOptions + stream?: boolean // 会话插件补充身份代际检查，重试与提示消费同一请求有效性 isCurrent: () => boolean } diff --git a/frontend/src/api/request/plugins/raw.ts b/frontend/src/api/request/plugins/raw.ts index 6e6d2ac0f..5d59a6037 100644 --- a/frontend/src/api/request/plugins/raw.ts +++ b/frontend/src/api/request/plugins/raw.ts @@ -13,6 +13,7 @@ export interface RawResponse { } export interface RawResponseOptions { + // 原始字节或宿主错误的正文上限；成功流由消费方约束，不在请求层缓冲 maximumBytes: number accept: (response: RawResponseMeta) => boolean } @@ -23,16 +24,31 @@ export async function rawPlugin(context: RequestContext, next: RequestNext) { return next() context.config = { ...context.config, - responseType: 'arraybuffer', + responseType: context.stream ? 'stream' : 'arraybuffer', validateStatus: () => true, } const response = await next() - const body = response.data as ArrayBuffer const meta: RawResponseMeta = { status: response.status, header: name => rawHeader(response.headers, name), } if (options.accept(meta)) { + if (context.stream) { + const headers = new Headers() + for (const [name, value] of Object.entries(response.headers)) { + if (value != null) + headers.set(name, String(value)) + } + return { + ...response, + data: new Response(response.data as ReadableStream | null, { + status: response.status, + statusText: response.statusText, + headers, + }), + } + } + const body = response.data as ArrayBuffer if (body.byteLength > options.maximumBytes) throw new ApiError('响应内容超过大小限制', 502, undefined, rawHeader(response.headers, 'x-request-id'), 'http') return { @@ -45,6 +61,9 @@ export async function rawPlugin(context: RequestContext, next: RequestNext) { } } + const body = context.stream + ? await readErrorBody(response.data as ReadableStream | null, options.maximumBytes) + : response.data as ArrayBuffer const decoded = decodeRawEnvelope(body) const normalized = normalizeApiResponseError({ ...response, data: decoded }) if (normalized) @@ -58,6 +77,35 @@ export async function rawPlugin(context: RequestContext, next: RequestNext) { ) } +async function readErrorBody(body: ReadableStream | null, maximumBytes: number) { + const reader = body?.getReader() + if (!reader) + return new ArrayBuffer(0) + const chunks: Uint8Array[] = [] + let length = 0 + try { + for (;;) { + const { done, value } = await reader.read() + if (done) + break + length += value.byteLength + if (length > maximumBytes) + throw new ApiError('响应内容超过大小限制', 502, undefined, undefined, 'http') + chunks.push(value) + } + } + finally { + reader.releaseLock() + } + const bytes = new Uint8Array(length) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return bytes.buffer +} + function rawHeader(headers: unknown, name: string) { if (!headers || typeof headers !== 'object') return undefined diff --git a/frontend/src/api/request/plugins/session.ts b/frontend/src/api/request/plugins/session.ts index 3100c071a..eed0b87e6 100644 --- a/frontend/src/api/request/plugins/session.ts +++ b/frontend/src/api/request/plugins/session.ts @@ -68,9 +68,16 @@ export async function sessionPlugin(context: RequestContext, send: RequestNext) return expireSession(new ApiError('登录已失效，请重新登录', 401, SESSION_REQUIRED), config, generation) if (config.signal?.aborted || generation !== sessionGeneration) throw new ApiError('请求已取消', 0, undefined, undefined, 'cancelled') + const sendCurrent: RequestNext = async () => { + const response = await send() + if (context.isCurrent()) + return response + // 身份已切换时丢弃迟到结果，流式传输由请求层终止 + throw new ApiError('请求已取消', 0, undefined, undefined, 'cancelled') + } const recovered = successfulRecoveries try { - return await send() + return await sendCurrent() } catch (error) { if (!isSessionRequired(error) || config.signal?.aborted || generation !== sessionGeneration) @@ -81,7 +88,7 @@ export async function sessionPlugin(context: RequestContext, send: RequestNext) if (config.signal?.aborted || generation !== sessionGeneration) throw error try { - return await send() + return await sendCurrent() } catch (retryError) { if (!isSessionRequired(retryError)) diff --git a/frontend/src/views/keys/composables/useApiKeyUse.ts b/frontend/src/views/keys/composables/useApiKeyUse.ts index 33e15bd36..38805ad4a 100644 --- a/frontend/src/views/keys/composables/useApiKeyUse.ts +++ b/frontend/src/views/keys/composables/useApiKeyUse.ts @@ -1,15 +1,17 @@ import type { ApiKey } from '@/api' import { ZNotification } from '@codex-proxy/ui' -import { shallowRef } from 'vue' +import { onScopeDispose, shallowRef } from 'vue' import { revealApiKey } from '@/api' import { useCopyText } from '@/composables/useCopyText' import { useIdSet } from '@/composables/useIdSet' +import { useRequestState } from '@/composables/useRequestState' import { buildCodexCcSwitchImportDeeplink, resolveServiceRootUrl } from '@/utils/client' // 明文读取、复制、展示与导入共用生命周期，列表数据不保存明文。 export function useApiKeyUse() { const copyText = useCopyText() const revealingKeys = useIdSet() + const revealRequest = useRequestState() const showUseKeyModal = shallowRef(false) const selectedUseKey = shallowRef<(ApiKey & { key: string }) | null>(null) const showCreatedKeyModal = shallowRef(false) @@ -35,22 +37,20 @@ export function useApiKeyUse() { }) } - async function openUseKeyModal(apiKey: ApiKey) { - const key = await revealPlaintextKey(apiKey) - if (!key) - return - selectedUseKey.value = { ...apiKey, key } - showUseKeyModal.value = true + function openUseKeyModal(apiKey: ApiKey) { + return withPlaintextKey(apiKey, (key) => { + selectedUseKey.value = { ...apiKey, key } + showUseKeyModal.value = true + }) } - async function importToCcs(apiKey: ApiKey) { - const key = await revealPlaintextKey(apiKey) - if (!key) - return - window.location.href = buildCodexCcSwitchImportDeeplink({ - apiKey: key, - baseUrl: openAiBaseUrl, - providerName: apiKey.name || apiKey.prefix || 'codex-proxy-rs', + function importToCcs(apiKey: ApiKey) { + return withPlaintextKey(apiKey, (key) => { + window.location.href = buildCodexCcSwitchImportDeeplink({ + apiKey: key, + baseUrl: openAiBaseUrl, + providerName: apiKey.name || apiKey.prefix || 'codex-proxy-rs', + }) }) } @@ -58,26 +58,31 @@ export function useApiKeyUse() { await copyText(text, { successText: '已复制到剪贴板', emptyErrorText: '复制失败' }) } - async function revealPlaintextKey(apiKey: ApiKey) { + async function withPlaintextKey(apiKey: ApiKey, use: (key: string) => void | Promise) { + if (revealingKeys.has(apiKey.id)) + return + // 明文操作共用一个当前意图，跨行点击和页面离开都使旧结果失效 + const requestId = revealRequest.start() try { - const result = await revealingKeys.run(apiKey.id, () => revealApiKey({ id: apiKey.id })) - if (!result) - return undefined + const result = await revealingKeys.run(apiKey.id, () => revealApiKey({ id: apiKey.id }, { signal: revealRequest.signal })) + if (!result || !revealRequest.isCurrent(requestId)) + return if (!result.plaintextKey) { ZNotification.error({ message: '完整 API Key 不可用' }) - return undefined + return } - return result.plaintextKey + await use(result.plaintextKey) } - catch { - return undefined + catch (error) { + revealRequest.fail(requestId, error) + } + finally { + revealRequest.finish(requestId) } } - async function copyApiKey(apiKey: ApiKey) { - const key = await revealPlaintextKey(apiKey) - if (key) - await copyToClipboard(key) + function copyApiKey(apiKey: ApiKey) { + return withPlaintextKey(apiKey, copyToClipboard) } function clearCreatedKey() { @@ -85,6 +90,11 @@ export function useApiKeyUse() { createdKeyName.value = '' } + onScopeDispose(() => { + selectedUseKey.value = null + clearCreatedKey() + }) + return { showUseKeyModal, selectedUseKey, diff --git a/frontend/src/views/plugins/components/PluginEnableModal.vue b/frontend/src/views/plugins/components/PluginEnableModal.vue index 77c103ddb..059ef4dea 100644 --- a/frontend/src/views/plugins/components/PluginEnableModal.vue +++ b/frontend/src/views/plugins/components/PluginEnableModal.vue @@ -12,7 +12,7 @@ const props = defineProps<{ replacements: (Pick & { version?: string })[] saving: boolean }>() -defineEmits<{ confirm: [] }>() +defineEmits<{ confirm: [], closed: [] }>() const open = defineModel({ required: true }) const parameters = computed(() => { const schema = props.metadata?.configurationSchema as JsonSchema | undefined @@ -27,7 +27,7 @@ const parameters = computed(() => {