mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-09 16:06:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c8618aa0f | ||
|
|
c43a8872db | ||
|
|
51c5caed0e | ||
|
|
a7f95c3a78 | ||
|
|
421e37896a | ||
|
|
1d1ccda42a | ||
|
|
e38cf31323 | ||
|
|
614c81c375 | ||
|
|
f0484c4e11 | ||
|
|
dffc417b9f | ||
|
|
2d4294c3c9 | ||
|
|
3d9906facf | ||
|
|
da9c2b92b0 | ||
|
|
de5c10c4e4 | ||
|
|
939cb99985 | ||
|
|
2bb8ebc01c | ||
|
|
6a5300a7b5 | ||
|
|
2c5b8b5c1a | ||
|
|
0a87ac7f76 | ||
|
|
3ac9d5f6f8 | ||
|
|
e7e5b659cd | ||
|
|
b225b3732d | ||
|
|
9dca763ea4 |
@@ -108,7 +108,7 @@ jobs:
|
||||
components: rustfmt,clippy
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
|
||||
@@ -29,7 +29,6 @@ jobs:
|
||||
backend: ${{ steps.filter.outputs.backend }}
|
||||
frontend: ${{ steps.filter.outputs.frontend }}
|
||||
container: ${{ steps.filter.outputs.container }}
|
||||
source: ${{ steps.filter.outputs.source }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -41,7 +40,7 @@ jobs:
|
||||
id: filter
|
||||
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||
with:
|
||||
# 目录规则取并集,再统一排除说明文档,保留隐藏配置与子模块指针
|
||||
# 目录规则取并集,再统一排除说明文档,保留隐藏配置
|
||||
predicate-quantifier: some-with-excludes
|
||||
filters: |
|
||||
backend:
|
||||
@@ -61,17 +60,6 @@ jobs:
|
||||
- '.github/workflows/ci.yml'
|
||||
- '.github/workflows/_quality.yml'
|
||||
- '!**/*.md'
|
||||
source:
|
||||
- '.github/scripts/toolchains.rb'
|
||||
- 'backend/rust-toolchain.toml'
|
||||
- 'deploy/Dockerfile'
|
||||
- '.gitmodules'
|
||||
- 'modules/ui'
|
||||
- 'modules/plugins'
|
||||
- 'frontend/**'
|
||||
- 'backend/crates/gateway-plugin/sdk/**'
|
||||
- '.github/workflows/ci.yml'
|
||||
- '!**/*.md'
|
||||
container:
|
||||
- 'backend/**'
|
||||
- 'frontend/**'
|
||||
@@ -101,56 +89,3 @@ jobs:
|
||||
uses: ./.github/workflows/_container.yml
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
source-development:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.source == 'true'
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout with development modules
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
submodules: recursive
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
package_json_file: frontend/package.json
|
||||
- name: Read build toolchains
|
||||
id: toolchains
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
||||
with:
|
||||
toolchain: ${{ steps.toolchains.outputs.rust }}
|
||||
- name: Install development dependencies
|
||||
run: |
|
||||
pnpm --dir modules/ui install --frozen-lockfile
|
||||
pnpm --dir frontend install --frozen-lockfile
|
||||
pnpm --dir modules/plugins/examples/workbench/frontend install --frozen-lockfile
|
||||
- name: Check UI source integration
|
||||
run: |
|
||||
pnpm --dir frontend build:source
|
||||
pnpm --dir modules/plugins/examples/workbench/frontend build:source
|
||||
- name: Check local SDK integration
|
||||
env:
|
||||
RUST_MIN_STACK: 16777216
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/backend/target/plugin-development
|
||||
run: |
|
||||
sdk_check_dir="$RUNNER_TEMP/plugin-source"
|
||||
mkdir -p "$sdk_check_dir/backend"
|
||||
cp modules/plugins/examples/workbench/backend/Cargo.{toml,lock} "$sdk_check_dir/backend/"
|
||||
cp -R modules/plugins/examples/workbench/backend/{src,tests} "$sdk_check_dir/backend/"
|
||||
cp modules/plugins/examples/workbench/plugin.json "$sdk_check_dir/"
|
||||
cargo test --manifest-path "$sdk_check_dir/backend/Cargo.toml" \
|
||||
--config "patch.\"https://github.com/zyycn/codex-proxy-rs.git\".gateway-plugin-sdk.path=\"$GITHUB_WORKSPACE/backend/crates/gateway-plugin/sdk\""
|
||||
- name: Ensure source checks preserve lockfiles
|
||||
run: |
|
||||
git diff --exit-code -- frontend/package.json frontend/pnpm-lock.yaml backend/Cargo.lock
|
||||
git -C modules/plugins diff --exit-code -- examples/workbench/frontend/package.json examples/workbench/frontend/pnpm-lock.yaml examples/workbench/backend/Cargo.lock
|
||||
|
||||
@@ -312,7 +312,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -409,7 +409,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
|
||||
+1
-1
@@ -228,7 +228,7 @@ git diff --check
|
||||
|
||||
## 自动检查与合并条件
|
||||
|
||||
[CI](.github/workflows/ci.yml) 按变更范围调用质量检查,Markdown 不触发后端、前端、容器和源码联调任务,工作流语法检查独立执行
|
||||
[CI](.github/workflows/ci.yml) 按变更范围调用质量检查,Markdown 不触发后端、前端和容器任务,工作流语法检查独立执行
|
||||
|
||||
后端 CI 使用 nextest 执行 `main` 集成测试,默认 Redis 密码模式执行完整测试集,ACL 用户模式复用同一批构建产物执行 Store 的 Redis 测试;本地验证命令见 [架构文档](docs/architecture.md#验证命令)
|
||||
|
||||
|
||||
Generated
+90
-100
@@ -147,9 +147,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.50"
|
||||
version = "0.4.51"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1"
|
||||
checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
@@ -224,9 +224,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "aws-credential-types"
|
||||
version = "1.3.0"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e"
|
||||
checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -260,9 +260,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.10.0"
|
||||
version = "1.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa"
|
||||
checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -286,9 +286,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.152.0"
|
||||
version = "1.155.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "004d5acbd6850ab58789a43822e0a2ef23927a46b012c17f8fa51253103db9f8"
|
||||
checksum = "5a542f0e9261b9ce56708d591a62062079fa3c58a362ffbf49a2e48a4f6ced16"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.6.0"
|
||||
version = "1.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b"
|
||||
checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -344,9 +344,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-async"
|
||||
version = "1.3.0"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316"
|
||||
checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
@@ -376,9 +376,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.61.4"
|
||||
version = "0.61.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609"
|
||||
checksum = "14c8a643d0e114cae39e45415e3225793a6396698b7779a0c9e3ede731d8fc71"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
@@ -387,9 +387,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http"
|
||||
version = "0.64.1"
|
||||
version = "0.64.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1"
|
||||
checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398"
|
||||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -403,15 +403,14 @@ dependencies = [
|
||||
"http-body-util",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.5.0"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d"
|
||||
checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -433,9 +432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.63.1"
|
||||
version = "0.65.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18"
|
||||
checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -453,9 +452,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.16.0"
|
||||
version = "1.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba"
|
||||
checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -470,16 +469,15 @@ dependencies = [
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.19.0"
|
||||
version = "1.19.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4"
|
||||
checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -505,9 +503,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "0.2.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829"
|
||||
checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -516,9 +514,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1"
|
||||
checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -530,7 +528,6 @@ dependencies = [
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"ryu",
|
||||
"serde",
|
||||
"time",
|
||||
@@ -540,9 +537,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-xml"
|
||||
version = "0.62.1"
|
||||
version = "0.64.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf"
|
||||
checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
@@ -552,9 +549,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc"
|
||||
checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
@@ -945,9 +942,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.45"
|
||||
version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a"
|
||||
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
|
||||
dependencies = [
|
||||
"compression-core",
|
||||
"flate2",
|
||||
@@ -956,9 +953,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "compression-core"
|
||||
version = "0.4.33"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
|
||||
checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a"
|
||||
|
||||
[[package]]
|
||||
name = "config"
|
||||
@@ -1084,9 +1081,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||
checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
@@ -1166,14 +1163,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.37.0"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98"
|
||||
checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa",
|
||||
"phf 0.13.1",
|
||||
"phf 0.14.0",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
@@ -2125,12 +2122,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.39.0"
|
||||
version = "0.40.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
|
||||
checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2483,9 +2481,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "0.58.6"
|
||||
version = "0.60.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1"
|
||||
checksum = "7e7305938cac5ee144352ceb71583ffc6cfeac7d20fcb20ef865a4d5c314e499"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
@@ -2511,23 +2509,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-regex"
|
||||
version = "0.58.6"
|
||||
version = "0.60.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a968fffd387b2ab005bf00c1851f5d26e90df05b6f7ad609ed0d620033cca5e7"
|
||||
checksum = "6f8a848331614144c03e815fb931f238c9e342bd7ab501b29e35d647e7cd5688"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-value"
|
||||
version = "0.58.6"
|
||||
version = "0.60.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96"
|
||||
checksum = "5a5a947ef798eeacacb43edf29b1d0ab64bdf5e2c9fbcaad10ee8663d922b544"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bytecount",
|
||||
"fraction",
|
||||
"getrandom 0.3.4",
|
||||
"memchr",
|
||||
"num-cmp",
|
||||
"num-traits",
|
||||
"serde_json",
|
||||
@@ -2642,9 +2641,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.39.0"
|
||||
version = "0.40.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de"
|
||||
checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril",
|
||||
@@ -3101,23 +3100,23 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
|
||||
dependencies = [
|
||||
"phf_macros 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_macros 0.14.0",
|
||||
"phf_shared 0.14.0",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
|
||||
checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_generator 0.14.0",
|
||||
"phf_shared 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3132,12 +3131,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_shared 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3155,12 +3154,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_generator 0.14.0",
|
||||
"phf_shared 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -3186,9 +3185,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
|
||||
checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
@@ -3199,12 +3198,6 @@ version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0"
|
||||
@@ -3636,9 +3629,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.58.6"
|
||||
version = "0.60.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f"
|
||||
checksum = "922268812b6f9e20b4ab0f1efe8e07036958eed7346ceddb31c777b5531fd11f"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"fluent-uri",
|
||||
@@ -3875,9 +3868,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.27.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2"
|
||||
checksum = "6844b5f49a5579f12ba4f89d9f7eec6d71889e4aa0bc7181821040ce66f08fff"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
@@ -3885,7 +3878,6 @@ dependencies = [
|
||||
"html5ever",
|
||||
"precomputed-hash",
|
||||
"selectors",
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3950,17 +3942,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.38.0"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d"
|
||||
checksum = "8322aebea94a6cc288ff884e8a49bdbceef60206c400c6c46d22e62fc4fd91a6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen",
|
||||
"precomputed-hash",
|
||||
"rustc-hash",
|
||||
"servo_arc",
|
||||
@@ -4067,9 +4057,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.3"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
|
||||
checksum = "b4eceb519ec34d8290aaf6e46f435102f2bfa63f22b05a144fb16ab266d3b198"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
@@ -4401,24 +4391,24 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.9.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
|
||||
checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_shared 0.14.0",
|
||||
"precomputed-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.6.1"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
|
||||
checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"phf_generator 0.14.0",
|
||||
"phf_shared 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
@@ -5214,11 +5204,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.6"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3"
|
||||
checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada"
|
||||
dependencies = [
|
||||
"phf 0.13.1",
|
||||
"phf 0.14.0",
|
||||
"phf_codegen",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
@@ -5651,9 +5641,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.0.4"
|
||||
version = "9.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22"
|
||||
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
|
||||
@@ -103,6 +103,10 @@ async fn execute(
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
or_shutdown!(plugin_runtime, management.bind(&plugin_runtime));
|
||||
let core = or_shutdown!(plugin_runtime, core.activate().await);
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_client_identity_port(&core.client_identity_port())
|
||||
);
|
||||
let nested_models = core.nested_model_execution_port();
|
||||
let affinity_lookup = core.affinity_lookup_port();
|
||||
or_shutdown!(
|
||||
|
||||
@@ -38,6 +38,10 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
or_shutdown!(plugin_runtime, management.bind(&plugin_runtime));
|
||||
let mut core = or_shutdown!(plugin_runtime, core.activate().await);
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_client_identity_port(&core.client_identity_port())
|
||||
);
|
||||
let nested_models = core.nested_model_execution_port();
|
||||
let affinity_lookup = core.affinity_lookup_port();
|
||||
or_shutdown!(
|
||||
|
||||
@@ -370,10 +370,28 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_validate_openai_residency() {
|
||||
fn config_loader_should_accept_optional_openai_routing_strings() {
|
||||
for value in [
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!("us"),
|
||||
serde_json::json!("custom-value"),
|
||||
serde_json::json!(""),
|
||||
] {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["residency"] = value.clone();
|
||||
document["openai"]["account_routing_override"] = value;
|
||||
parse_config(&document.to_string()).expect("optional routing strings");
|
||||
}
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["residency"] = serde_json::json!("invalid");
|
||||
assert_rejected(document.to_string());
|
||||
document["openai"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("residency");
|
||||
document["openai"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("account_routing_override");
|
||||
parse_config(&document.to_string()).expect("omitted routing strings");
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -84,6 +84,7 @@ pub struct PluginInstanceMutation {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PluginInstanceRuntimeStatus {
|
||||
Disabled,
|
||||
Unknown,
|
||||
AwaitingPublication,
|
||||
Preparing,
|
||||
Running,
|
||||
@@ -108,6 +109,21 @@ pub struct PluginInstanceRuntime {
|
||||
pub actual_artifact_sha256: Option<String>,
|
||||
pub failure: Option<PluginInstanceRuntimeFailure>,
|
||||
pub draining_revisions: Vec<u64>,
|
||||
pub load: Option<PluginInstanceRuntimeLoad>,
|
||||
}
|
||||
|
||||
/// 当前发布会话的资源占用采样,计数可以重叠且不构成原子快照
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PluginInstanceRuntimeLoad {
|
||||
/// 总在途名额,包含排队、取消确认和仍由响应流持有的调用
|
||||
pub active_calls: usize,
|
||||
pub executing_calls: usize,
|
||||
pub queued_calls: usize,
|
||||
pub resuming_calls: usize,
|
||||
/// 包含已返回但尚未由插件消费的回调结果
|
||||
pub pending_callbacks: usize,
|
||||
/// 已交付首结果但尚未收到 End 的流
|
||||
pub active_streams: usize,
|
||||
}
|
||||
|
||||
/// 可继续运行的旧接口提示,与无法启用的兼容性错误分开
|
||||
|
||||
@@ -1,773 +0,0 @@
|
||||
//! 插件实例的配置、停用、删除与私有状态迁移编排
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use gateway_core::{
|
||||
policy::ClientApiKeyId,
|
||||
routing::{AccountGroupId, extensions::ExtensionSetReference},
|
||||
};
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
use super::PluginsService;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind, MutationContext, Revision,
|
||||
plugins::instances::{
|
||||
ConfigurePluginInstance, PluginInstance, PluginInstanceMutation,
|
||||
PluginInstanceReplacement, PluginInstanceRuntime, PluginInstanceRuntimeStatus,
|
||||
PluginInstanceSnapshot, PluginInstanceView,
|
||||
},
|
||||
plugins::state::{PluginStateCommit, PluginStateConfiguration},
|
||||
},
|
||||
use_case::{map_store_error, publish_committed},
|
||||
};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct PackageStatus {
|
||||
warning: Option<String>,
|
||||
load_error: Option<String>,
|
||||
}
|
||||
|
||||
impl PluginsService {
|
||||
/// 只校验固定包体,不启动插件;停用实例也需区分兼容性提醒与加载错误
|
||||
async fn package_status(&self, digest: &str) -> Result<PackageStatus, AdminError> {
|
||||
// 运行宿主和摘要固定,缓存纯静态结论,避免管理页轮询反复解包
|
||||
// 只缓存有限数量的确定结果,暂时性错误仍可重试
|
||||
let mut cache = self.compatibility.lock().await;
|
||||
if let Some(warning) = cache.get(digest) {
|
||||
return Ok(warning.clone());
|
||||
}
|
||||
let artifact = self
|
||||
.store
|
||||
.load_artifact(digest)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
let warning = match self
|
||||
.inspector
|
||||
.compatibility_warning(artifact.archive, digest.to_owned())
|
||||
.await
|
||||
{
|
||||
Ok(warning) => PackageStatus {
|
||||
warning,
|
||||
load_error: None,
|
||||
},
|
||||
Err(error) if error.kind() == AdminErrorKind::Invalid => PackageStatus {
|
||||
warning: None,
|
||||
load_error: Some(error.message().to_owned()),
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if cache.len() < 128 {
|
||||
cache.insert(digest.to_owned(), warning.clone());
|
||||
}
|
||||
Ok(warning)
|
||||
}
|
||||
|
||||
pub async fn instances(&self) -> Result<Vec<PluginInstanceView>, AdminError> {
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let published = self.published.snapshot_for_diagnostics();
|
||||
let published_revision = published.as_ref().map(|snapshot| snapshot.revision().get());
|
||||
let published_ready = published
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.extensions().is_none_or(|set| set.can_serve()));
|
||||
let mut diagnostics = self
|
||||
.preparation
|
||||
.runtime_diagnostics(
|
||||
&snapshot,
|
||||
published_revision,
|
||||
published
|
||||
.as_ref()
|
||||
.and_then(|snapshot| snapshot.extensions()),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let target_revision = snapshot.config_revision.get();
|
||||
let artifacts = self
|
||||
.list()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|artifact| (artifact.metadata.sha256.clone(), artifact.metadata))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut views = Vec::with_capacity(snapshot.instances.len());
|
||||
for instance in snapshot.instances {
|
||||
let metadata = artifacts
|
||||
.get(&instance.artifact_sha256)
|
||||
.ok_or_else(|| AdminError::not_found("插件制品不存在"))?;
|
||||
let package_status = self.package_status(&instance.artifact_sha256).await?;
|
||||
let configuration_required = package_status.load_error.is_none()
|
||||
&& !self
|
||||
.preparation
|
||||
.configuration_ready(instance.clone(), metadata)
|
||||
.await?;
|
||||
let runtime = diagnostics.remove(&instance.id).unwrap_or_else(|| {
|
||||
let running = instance.enabled
|
||||
&& published_ready
|
||||
&& published_revision == Some(target_revision);
|
||||
PluginInstanceRuntime {
|
||||
status: if !instance.enabled {
|
||||
PluginInstanceRuntimeStatus::Disabled
|
||||
} else if running {
|
||||
PluginInstanceRuntimeStatus::Running
|
||||
} else {
|
||||
PluginInstanceRuntimeStatus::AwaitingPublication
|
||||
},
|
||||
actual_revision: running.then_some(instance.revision.get()),
|
||||
actual_artifact_sha256: running.then(|| instance.artifact_sha256.clone()),
|
||||
failure: None,
|
||||
draining_revisions: Vec::new(),
|
||||
}
|
||||
});
|
||||
views.push(PluginInstanceView {
|
||||
api_deprecations: if package_status.load_error.is_none() {
|
||||
self.inspector.api_deprecations(metadata)?
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
configuration_required,
|
||||
compatibility_warning: package_status.warning,
|
||||
load_error: package_status.load_error,
|
||||
running: runtime.status == PluginInstanceRuntimeStatus::Running,
|
||||
published_revision,
|
||||
runtime,
|
||||
instance,
|
||||
});
|
||||
}
|
||||
Ok(views)
|
||||
}
|
||||
|
||||
pub async fn configure_instance(
|
||||
&self,
|
||||
id: Option<&str>,
|
||||
input: ConfigurePluginInstance,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
if id.is_some() && input.creation_id.is_some() {
|
||||
return Err(AdminError::invalid("编辑配置不能使用创建标识"));
|
||||
}
|
||||
if id.is_none() && input.expected_revision.is_some() {
|
||||
return Err(AdminError::invalid("创建配置不能指定已有配置版本"));
|
||||
}
|
||||
if let Some(creation_id) = &input.creation_id {
|
||||
let parsed = uuid::Uuid::parse_str(creation_id)
|
||||
.map_err(|_| AdminError::invalid("配置创建标识必须是 UUID"))?;
|
||||
if parsed.to_string() != *creation_id {
|
||||
return Err(AdminError::invalid("配置创建标识必须使用标准 UUID 格式"));
|
||||
}
|
||||
}
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let existing = match id {
|
||||
Some(id) => Some(
|
||||
snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.cloned()
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?,
|
||||
),
|
||||
None => input.creation_id.as_ref().and_then(|creation_id| {
|
||||
snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == *creation_id)
|
||||
.cloned()
|
||||
}),
|
||||
};
|
||||
if let Some(existing) = &existing {
|
||||
if id.is_none() && !same_creation_request(existing, &input) {
|
||||
return Err(AdminError::conflict("此配置已创建,请加载最新配置后编辑"));
|
||||
}
|
||||
if input
|
||||
.expected_revision
|
||||
.is_some_and(|revision| revision != existing.revision.get())
|
||||
{
|
||||
return Err(AdminError::conflict("插件配置已变更,请加载最新配置后重试"));
|
||||
}
|
||||
}
|
||||
self.configure_from_snapshot(snapshot, existing, input, context)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn configure_from_snapshot(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
existing: Option<PluginInstance>,
|
||||
input: ConfigurePluginInstance,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let expected_revision = snapshot.config_revision;
|
||||
let candidate_revision = next_revision(expected_revision)?;
|
||||
let artifacts = self
|
||||
.store
|
||||
.list_artifacts()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
let artifact = artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.metadata.sha256 == input.artifact_sha256)
|
||||
.ok_or_else(|| AdminError::not_found("插件制品不存在"))?;
|
||||
if artifact.accepted_at.is_none() {
|
||||
return Err(AdminError::invalid("请先安装并接受该版本声明的权限"));
|
||||
}
|
||||
if input.enabled
|
||||
&& let Some(warning) = self
|
||||
.package_status(&input.artifact_sha256)
|
||||
.await?
|
||||
.load_error
|
||||
{
|
||||
return Err(AdminError::invalid(format!(
|
||||
"{warning},无法启动,请安装兼容版本"
|
||||
)));
|
||||
}
|
||||
let same_plugin = |digest: &str| {
|
||||
artifacts.iter().any(|item| {
|
||||
item.metadata.sha256 == digest
|
||||
&& item.metadata.plugin_id == artifact.metadata.plugin_id
|
||||
})
|
||||
};
|
||||
if existing
|
||||
.as_ref()
|
||||
.is_some_and(|previous| !same_plugin(&previous.artifact_sha256))
|
||||
{
|
||||
return Err(AdminError::invalid("不能将已有配置切换为另一个插件"));
|
||||
}
|
||||
if input.enabled
|
||||
&& snapshot.instances.iter().any(|other| {
|
||||
other.enabled
|
||||
&& existing
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.id != other.id)
|
||||
&& same_plugin(&other.artifact_sha256)
|
||||
&& !input
|
||||
.replace_instances
|
||||
.iter()
|
||||
.any(|replacement| replacement.id == other.id)
|
||||
})
|
||||
{
|
||||
return Err(AdminError::conflict(
|
||||
"此插件已有启用配置,请先切换或停用旧配置",
|
||||
));
|
||||
}
|
||||
let saved = if let Some(previous) = &existing
|
||||
&& previous.artifact_sha256 != input.artifact_sha256
|
||||
&& input.secrets.is_none()
|
||||
{
|
||||
self.store
|
||||
.load_version_configuration(&previous.id, &input.artifact_sha256)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let instance = PluginInstance {
|
||||
id: existing.as_ref().map_or_else(
|
||||
|| {
|
||||
input
|
||||
.creation_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string())
|
||||
},
|
||||
|instance| instance.id.clone(),
|
||||
),
|
||||
name: input.name.trim().to_owned(),
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: input.enabled,
|
||||
trusted_process: true,
|
||||
configuration: input.configuration,
|
||||
secrets: input
|
||||
.secrets
|
||||
.or_else(|| saved.map(|saved| saved.secrets))
|
||||
.unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
}),
|
||||
bindings: input.bindings,
|
||||
revision: candidate_revision,
|
||||
};
|
||||
validate_input(&instance)?;
|
||||
if existing
|
||||
.as_ref()
|
||||
.is_some_and(|previous| previous.artifact_sha256 != instance.artifact_sha256)
|
||||
&& !self
|
||||
.preparation
|
||||
.configuration_ready(instance.clone(), &artifact.metadata)
|
||||
.await?
|
||||
{
|
||||
return Err(AdminError::invalid(
|
||||
"目标版本仍缺少必填设置,请补充后再切换",
|
||||
));
|
||||
}
|
||||
|
||||
let replacements = input.replace_instances;
|
||||
if !replacements.is_empty() {
|
||||
if !instance.enabled || replacements.len() > 256 {
|
||||
return Err(AdminError::invalid(
|
||||
"仅启用配置时可以切换其他配置,单次最多 256 项",
|
||||
));
|
||||
}
|
||||
let mut ids = BTreeSet::new();
|
||||
for replacement in &replacements {
|
||||
if replacement.id == instance.id || !ids.insert(&replacement.id) {
|
||||
return Err(AdminError::invalid("停用列表不能重复或包含当前配置"));
|
||||
}
|
||||
let previous = snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|item| item.id == replacement.id)
|
||||
.ok_or_else(|| AdminError::conflict("待停用配置已变更,请重新确认"))?;
|
||||
if !previous.enabled || previous.revision.get() != replacement.expected_revision {
|
||||
return Err(AdminError::conflict("待停用配置已变更,请重新确认"));
|
||||
}
|
||||
if !artifacts.iter().any(|item| {
|
||||
item.metadata.sha256 == previous.artifact_sha256
|
||||
&& item.metadata.plugin_id == artifact.metadata.plugin_id
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能切换同一插件的配置"));
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = self.preparation.validate(instance.clone()).await?;
|
||||
if let Some(existing) = &existing
|
||||
&& self.state.transition_required(&existing.id, &state).await?
|
||||
{
|
||||
return self
|
||||
.configure_instance_with_state_transition(
|
||||
snapshot,
|
||||
existing.clone(),
|
||||
instance,
|
||||
state,
|
||||
&replacements,
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
snapshot.instances.retain(|item| item.id != instance.id);
|
||||
snapshot.instances.push(instance.clone());
|
||||
snapshot.config_revision = candidate_revision;
|
||||
apply_replacements(&mut snapshot, &replacements);
|
||||
let prepared = self.preparation.prepare(snapshot.clone()).await?;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance_replacing(
|
||||
instance,
|
||||
expected_revision,
|
||||
PluginStateCommit {
|
||||
configuration: state,
|
||||
transition_id: None,
|
||||
},
|
||||
&replacements,
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
self.publish_instance(snapshot, prepared, &result, &replacements)
|
||||
.await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn configure_instance_with_state_transition(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
previous: PluginInstance,
|
||||
mut target: PluginInstance,
|
||||
target_state: PluginStateConfiguration,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let previous_state = self.preparation.validate(previous.clone()).await?;
|
||||
let original_revision = snapshot.config_revision;
|
||||
// 即使最终保持停用,迁移也必须由目标制品的受控候选执行
|
||||
let mut migration_instance = target.clone();
|
||||
migration_instance.enabled = true;
|
||||
snapshot.config_revision = migration_instance.revision;
|
||||
snapshot
|
||||
.instances
|
||||
.retain(|instance| instance.id != migration_instance.id);
|
||||
snapshot.instances.push(migration_instance);
|
||||
apply_replacements(&mut snapshot, replacements);
|
||||
let migration_prepared = self.preparation.prepare(snapshot).await?;
|
||||
|
||||
let mut disabled_revision = None;
|
||||
let disabled = if previous.enabled {
|
||||
let mut disabled = previous.clone();
|
||||
disabled.enabled = false;
|
||||
let mutation = self
|
||||
.store
|
||||
.save_instance_with_state(
|
||||
disabled,
|
||||
original_revision,
|
||||
PluginStateCommit {
|
||||
configuration: previous_state.clone(),
|
||||
transition_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), mutation.config_revision).await?;
|
||||
self.preparation
|
||||
.quiesce_instance(&previous.id, &previous.artifact_sha256, previous.revision)
|
||||
.await;
|
||||
disabled_revision = Some(mutation.config_revision);
|
||||
mutation.instance
|
||||
} else {
|
||||
previous.clone()
|
||||
};
|
||||
let expected_revision = disabled_revision.unwrap_or(original_revision);
|
||||
let transition = match self
|
||||
.state
|
||||
.begin_transition(
|
||||
&disabled.id,
|
||||
disabled.revision,
|
||||
&target.artifact_sha256,
|
||||
target_state.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(transition) => transition,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.recover_failed_state_transition(
|
||||
None,
|
||||
&previous,
|
||||
&previous_state,
|
||||
disabled_revision,
|
||||
context,
|
||||
error,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
// 迁移开始后的所有提交前失败都走同一恢复出口;提交成功后不再回滚旧配置
|
||||
let committed = async {
|
||||
self.preparation
|
||||
.migrate_state(&migration_prepared, transition.clone())
|
||||
.await?;
|
||||
let mut current = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
if current.config_revision != expected_revision {
|
||||
return Err(AdminError::conflict("迁移期间插件配置已发生变化"));
|
||||
}
|
||||
let candidate_revision = next_revision(expected_revision)?;
|
||||
target.revision = candidate_revision;
|
||||
current.instances.retain(|item| item.id != target.id);
|
||||
current.instances.push(target.clone());
|
||||
current.config_revision = candidate_revision;
|
||||
apply_replacements(&mut current, replacements);
|
||||
let prepared = self.preparation.prepare(current.clone()).await?;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance_replacing(
|
||||
target,
|
||||
expected_revision,
|
||||
PluginStateCommit {
|
||||
configuration: target_state,
|
||||
transition_id: Some(transition.id.clone()),
|
||||
},
|
||||
replacements,
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
Ok((current, prepared, result))
|
||||
}
|
||||
.await;
|
||||
let (current, prepared, result) = match committed {
|
||||
Ok(committed) => committed,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.recover_failed_state_transition(
|
||||
Some(&transition.id),
|
||||
&previous,
|
||||
&previous_state,
|
||||
disabled_revision,
|
||||
context,
|
||||
error,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
self.publish_instance(current, prepared, &result, replacements)
|
||||
.await?;
|
||||
drop(migration_prepared);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn recover_failed_state_transition(
|
||||
&self,
|
||||
transition_id: Option<&str>,
|
||||
previous: &PluginInstance,
|
||||
previous_state: &PluginStateConfiguration,
|
||||
disabled_revision: Option<Revision>,
|
||||
context: &MutationContext,
|
||||
original_error: AdminError,
|
||||
) -> AdminError {
|
||||
if let Some(transition_id) = transition_id {
|
||||
self.state.abort_transition(transition_id).await;
|
||||
}
|
||||
let Some(disabled_revision) = disabled_revision else {
|
||||
return original_error;
|
||||
};
|
||||
let restored = async {
|
||||
let mut snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
if snapshot.config_revision != disabled_revision {
|
||||
return Err(AdminError::conflict(
|
||||
"状态迁移失败;实例已保持停用,且配置已被其他操作修改",
|
||||
));
|
||||
}
|
||||
let candidate_revision = next_revision(disabled_revision)?;
|
||||
let mut restore = previous.clone();
|
||||
restore.enabled = true;
|
||||
restore.revision = candidate_revision;
|
||||
snapshot.instances.retain(|item| item.id != restore.id);
|
||||
snapshot.instances.push(restore.clone());
|
||||
snapshot.config_revision = candidate_revision;
|
||||
let prepared = self.preparation.prepare(snapshot.clone()).await?;
|
||||
let mutation = self
|
||||
.store
|
||||
.save_instance_with_state(
|
||||
restore,
|
||||
disabled_revision,
|
||||
PluginStateCommit {
|
||||
configuration: previous_state.clone(),
|
||||
transition_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
self.publish_instance(snapshot, prepared, &mutation, &[])
|
||||
.await?;
|
||||
Ok::<(), AdminError>(())
|
||||
}
|
||||
.await;
|
||||
match restored {
|
||||
Ok(()) => original_error,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
error_kind = ?error.kind(),
|
||||
"plugin state migration failed and the previous instance could not be restored"
|
||||
);
|
||||
AdminError::conflict("状态迁移失败;为避免覆盖并发配置,插件实例已保持停用")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 以事务返回的版本激活并发布,候选强引用必须覆盖整个提交后阶段
|
||||
async fn publish_instance(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
prepared: ExtensionSetReference,
|
||||
mutation: &PluginInstanceMutation,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
) -> Result<(), AdminError> {
|
||||
let prepared = if mutation.config_revision == snapshot.config_revision
|
||||
&& mutation.instance.revision == snapshot.config_revision
|
||||
{
|
||||
prepared
|
||||
} else {
|
||||
snapshot.config_revision = mutation.config_revision;
|
||||
snapshot
|
||||
.instances
|
||||
.retain(|item| item.id != mutation.instance.id);
|
||||
snapshot.instances.push(mutation.instance.clone());
|
||||
apply_replacements(&mut snapshot, replacements);
|
||||
self.preparation.prepare(snapshot).await?
|
||||
};
|
||||
self.preparation
|
||||
.activate_state(&prepared, &mutation.instance)
|
||||
.await?;
|
||||
publish_committed(self.snapshots.as_ref(), mutation.config_revision).await?;
|
||||
drop(prepared);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_instance(
|
||||
&self,
|
||||
id: &str,
|
||||
context: &MutationContext,
|
||||
) -> Result<Revision, AdminError> {
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let instance = snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?;
|
||||
if instance.enabled {
|
||||
return Err(AdminError::conflict("请先停用插件实例"));
|
||||
}
|
||||
let revision = self
|
||||
.store
|
||||
.delete_instance(id, snapshot.config_revision, context)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), revision).await?;
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
/// 紧急管理修复不要求损坏插件成功准备;提交后沿用现有发布与暂停合同
|
||||
pub async fn disable_instance(
|
||||
&self,
|
||||
id: &str,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let PluginInstanceSnapshot {
|
||||
config_revision,
|
||||
instances,
|
||||
} = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let mut instance = instances
|
||||
.into_iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?;
|
||||
instance.enabled = false;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance(instance, config_revision, context)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_replacements(
|
||||
snapshot: &mut PluginInstanceSnapshot,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
) {
|
||||
for instance in &mut snapshot.instances {
|
||||
if replacements.iter().any(|item| item.id == instance.id) {
|
||||
instance.enabled = false;
|
||||
instance.revision = snapshot.config_revision;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn same_creation_request(existing: &PluginInstance, input: &ConfigurePluginInstance) -> bool {
|
||||
let same_secrets = input
|
||||
.secrets
|
||||
.as_ref()
|
||||
.map_or(existing.secrets.is_empty(), |secrets| {
|
||||
secrets.len() == existing.secrets.len()
|
||||
&& secrets.iter().all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
})
|
||||
});
|
||||
existing.name == input.name.trim()
|
||||
&& existing.artifact_sha256 == input.artifact_sha256
|
||||
&& existing.enabled == input.enabled
|
||||
&& existing.configuration == input.configuration
|
||||
&& existing.bindings == input.bindings
|
||||
&& same_secrets
|
||||
}
|
||||
|
||||
fn validate_input(instance: &PluginInstance) -> Result<(), AdminError> {
|
||||
if instance.name.is_empty()
|
||||
|| instance.name.len() > 128
|
||||
|| instance.name.chars().any(char::is_control)
|
||||
|| instance.artifact_sha256.len() != 64
|
||||
|| !instance
|
||||
.artifact_sha256
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
|| !instance.configuration.is_object()
|
||||
|| serde_json::to_vec(&instance.configuration).map_or(true, |bytes| bytes.len() > 48 * 1024)
|
||||
|| instance.secrets.len() > 64
|
||||
|| instance.bindings.len() > 64
|
||||
{
|
||||
return Err(AdminError::invalid("插件实例配置不合法或超过大小限制"));
|
||||
}
|
||||
use secrecy::ExposeSecret as _;
|
||||
if instance
|
||||
.secrets
|
||||
.iter()
|
||||
.any(|(key, value)| key.len() > 64 || value.expose_secret().len() > 8192)
|
||||
|| instance
|
||||
.secrets
|
||||
.values()
|
||||
.map(|value| value.expose_secret().len())
|
||||
.sum::<usize>()
|
||||
> 48 * 1024
|
||||
{
|
||||
return Err(AdminError::invalid("插件敏感配置超过大小限制"));
|
||||
}
|
||||
if instance.bindings.iter().any(|binding| {
|
||||
binding.contribution.is_empty()
|
||||
|| binding.contribution.len() > 128
|
||||
|| !binding
|
||||
.contribution
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-'))
|
||||
|| binding.client_key_ids.len() > 256
|
||||
|| binding.account_group_ids.len() > 256
|
||||
|| binding.client_key_ids.iter().collect::<BTreeSet<_>>().len()
|
||||
!= binding.client_key_ids.len()
|
||||
|| binding
|
||||
.account_group_ids
|
||||
.iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= binding.account_group_ids.len()
|
||||
|| binding
|
||||
.client_key_ids
|
||||
.iter()
|
||||
.any(|id| ClientApiKeyId::new(id.clone()).is_err())
|
||||
|| binding
|
||||
.account_group_ids
|
||||
.iter()
|
||||
.any(|id| AccountGroupId::new(id.clone()).is_err())
|
||||
|| binding.identity_bindings.len() > 256
|
||||
|| binding
|
||||
.identity_bindings
|
||||
.iter()
|
||||
.map(|identity| &identity.principal)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= binding.identity_bindings.len()
|
||||
|| binding.identity_bindings.iter().any(|identity| {
|
||||
identity.principal.trim() != identity.principal
|
||||
|| identity.principal.is_empty()
|
||||
|| identity.principal.len() > 256
|
||||
|| identity.principal.chars().any(char::is_control)
|
||||
|| ClientApiKeyId::new(identity.client_key_id.clone()).is_err()
|
||||
})
|
||||
}) {
|
||||
return Err(AdminError::invalid("插件绑定的范围或入口认证身份不合法"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_revision(revision: Revision) -> Result<Revision, AdminError> {
|
||||
Revision::new(
|
||||
revision
|
||||
.get()
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| AdminError::internal("配置版本已耗尽"))?,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("配置版本不合法"))
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//! 插件配置的输入约束、创建幂等比较与候选快照变换
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use gateway_core::{policy::ClientApiKeyId, routing::AccountGroupId};
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
use crate::model::{
|
||||
AdminError, Revision,
|
||||
plugins::instances::{
|
||||
ConfigurePluginInstance, PluginInstance, PluginInstanceReplacement, PluginInstanceSnapshot,
|
||||
},
|
||||
};
|
||||
|
||||
pub(super) fn apply_replacements(
|
||||
snapshot: &mut PluginInstanceSnapshot,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
) {
|
||||
for instance in &mut snapshot.instances {
|
||||
if replacements.iter().any(|item| item.id == instance.id) {
|
||||
instance.enabled = false;
|
||||
instance.revision = snapshot.config_revision;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn same_creation_request(
|
||||
existing: &PluginInstance,
|
||||
input: &ConfigurePluginInstance,
|
||||
) -> bool {
|
||||
let same_secrets = input
|
||||
.secrets
|
||||
.as_ref()
|
||||
.map_or(existing.secrets.is_empty(), |secrets| {
|
||||
secrets.len() == existing.secrets.len()
|
||||
&& secrets.iter().all(|(key, value)| {
|
||||
existing
|
||||
.secrets
|
||||
.get(key)
|
||||
.is_some_and(|current| current.expose_secret() == value.expose_secret())
|
||||
})
|
||||
});
|
||||
existing.name == input.name.trim()
|
||||
&& existing.artifact_sha256 == input.artifact_sha256
|
||||
&& existing.enabled == input.enabled
|
||||
&& existing.configuration == input.configuration
|
||||
&& existing.bindings == input.bindings
|
||||
&& same_secrets
|
||||
}
|
||||
|
||||
pub(super) fn validate_input(instance: &PluginInstance) -> Result<(), AdminError> {
|
||||
if instance.name.is_empty()
|
||||
|| instance.name.len() > 128
|
||||
|| instance.name.chars().any(char::is_control)
|
||||
|| instance.artifact_sha256.len() != 64
|
||||
|| !instance
|
||||
.artifact_sha256
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
|| !instance.configuration.is_object()
|
||||
|| serde_json::to_vec(&instance.configuration).map_or(true, |bytes| bytes.len() > 48 * 1024)
|
||||
|| instance.secrets.len() > 64
|
||||
|| instance.bindings.len() > 64
|
||||
{
|
||||
return Err(AdminError::invalid("插件实例配置不合法或超过大小限制"));
|
||||
}
|
||||
use secrecy::ExposeSecret as _;
|
||||
if instance
|
||||
.secrets
|
||||
.iter()
|
||||
.any(|(key, value)| key.len() > 64 || value.expose_secret().len() > 8192)
|
||||
|| instance
|
||||
.secrets
|
||||
.values()
|
||||
.map(|value| value.expose_secret().len())
|
||||
.sum::<usize>()
|
||||
> 48 * 1024
|
||||
{
|
||||
return Err(AdminError::invalid("插件敏感配置超过大小限制"));
|
||||
}
|
||||
if instance.bindings.iter().any(|binding| {
|
||||
binding.contribution.is_empty()
|
||||
|| binding.contribution.len() > 128
|
||||
|| !binding
|
||||
.contribution
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-'))
|
||||
|| binding.client_key_ids.len() > 256
|
||||
|| binding.account_group_ids.len() > 256
|
||||
|| binding.client_key_ids.iter().collect::<BTreeSet<_>>().len()
|
||||
!= binding.client_key_ids.len()
|
||||
|| binding
|
||||
.account_group_ids
|
||||
.iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= binding.account_group_ids.len()
|
||||
|| binding
|
||||
.client_key_ids
|
||||
.iter()
|
||||
.any(|id| ClientApiKeyId::new(id.clone()).is_err())
|
||||
|| binding
|
||||
.account_group_ids
|
||||
.iter()
|
||||
.any(|id| AccountGroupId::new(id.clone()).is_err())
|
||||
|| binding.identity_bindings.len() > 256
|
||||
|| binding
|
||||
.identity_bindings
|
||||
.iter()
|
||||
.map(|identity| &identity.principal)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= binding.identity_bindings.len()
|
||||
|| binding.identity_bindings.iter().any(|identity| {
|
||||
identity.principal.trim() != identity.principal
|
||||
|| identity.principal.is_empty()
|
||||
|| identity.principal.len() > 256
|
||||
|| identity.principal.chars().any(char::is_control)
|
||||
|| ClientApiKeyId::new(identity.client_key_id.clone()).is_err()
|
||||
})
|
||||
}) {
|
||||
return Err(AdminError::invalid("插件绑定的范围或入口认证身份不合法"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn next_revision(revision: Revision) -> Result<Revision, AdminError> {
|
||||
Revision::new(
|
||||
revision
|
||||
.get()
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| AdminError::internal("配置版本已耗尽"))?,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("配置版本不合法"))
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
//! 插件私有状态迁移的排空、暂存提交与失败恢复编排
|
||||
|
||||
use super::{
|
||||
PluginsService,
|
||||
configuration::{apply_replacements, next_revision},
|
||||
};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
plugins::{
|
||||
instances::{
|
||||
PluginInstance, PluginInstanceMutation, PluginInstanceReplacement,
|
||||
PluginInstanceSnapshot,
|
||||
},
|
||||
state::{PluginStateCommit, PluginStateConfiguration},
|
||||
},
|
||||
},
|
||||
use_case::{map_store_error, publish_committed},
|
||||
};
|
||||
|
||||
impl PluginsService {
|
||||
pub(super) async fn configure_instance_with_state_transition(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
previous: PluginInstance,
|
||||
mut target: PluginInstance,
|
||||
target_state: PluginStateConfiguration,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let previous_state = self.preparation.validate(previous.clone()).await?;
|
||||
let original_revision = snapshot.config_revision;
|
||||
// 即使最终保持停用,迁移也必须由目标制品的受控候选执行
|
||||
let mut migration_instance = target.clone();
|
||||
migration_instance.enabled = true;
|
||||
snapshot.config_revision = migration_instance.revision;
|
||||
snapshot
|
||||
.instances
|
||||
.retain(|instance| instance.id != migration_instance.id);
|
||||
snapshot.instances.push(migration_instance);
|
||||
apply_replacements(&mut snapshot, replacements);
|
||||
let migration_prepared = self.preparation.prepare(snapshot).await?;
|
||||
|
||||
let mut disabled_revision = None;
|
||||
let disabled = if previous.enabled {
|
||||
let mut disabled = previous.clone();
|
||||
disabled.enabled = false;
|
||||
let mutation = self
|
||||
.store
|
||||
.save_instance_with_state(
|
||||
disabled,
|
||||
original_revision,
|
||||
PluginStateCommit {
|
||||
configuration: previous_state.clone(),
|
||||
transition_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), mutation.config_revision).await?;
|
||||
self.preparation
|
||||
.quiesce_instance(&previous.id, &previous.artifact_sha256, previous.revision)
|
||||
.await;
|
||||
disabled_revision = Some(mutation.config_revision);
|
||||
mutation.instance
|
||||
} else {
|
||||
previous.clone()
|
||||
};
|
||||
let expected_revision = disabled_revision.unwrap_or(original_revision);
|
||||
let transition = match self
|
||||
.state
|
||||
.begin_transition(
|
||||
&disabled.id,
|
||||
disabled.revision,
|
||||
&target.artifact_sha256,
|
||||
target_state.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(transition) => transition,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.recover_failed_state_transition(
|
||||
None,
|
||||
&previous,
|
||||
&previous_state,
|
||||
disabled_revision,
|
||||
context,
|
||||
error,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
// 迁移开始后的所有提交前失败都走同一恢复出口;提交成功后不再回滚旧配置
|
||||
let committed = async {
|
||||
self.preparation
|
||||
.migrate_state(&migration_prepared, transition.clone())
|
||||
.await?;
|
||||
let mut current = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
if current.config_revision != expected_revision {
|
||||
return Err(AdminError::conflict("迁移期间插件配置已发生变化"));
|
||||
}
|
||||
let candidate_revision = next_revision(expected_revision)?;
|
||||
target.revision = candidate_revision;
|
||||
current.instances.retain(|item| item.id != target.id);
|
||||
current.instances.push(target.clone());
|
||||
current.config_revision = candidate_revision;
|
||||
apply_replacements(&mut current, replacements);
|
||||
let prepared = self.preparation.prepare(current.clone()).await?;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance_replacing(
|
||||
target,
|
||||
expected_revision,
|
||||
PluginStateCommit {
|
||||
configuration: target_state,
|
||||
transition_id: Some(transition.id.clone()),
|
||||
},
|
||||
replacements,
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
Ok((current, prepared, result))
|
||||
}
|
||||
.await;
|
||||
let (current, prepared, result) = match committed {
|
||||
Ok(committed) => committed,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.recover_failed_state_transition(
|
||||
Some(&transition.id),
|
||||
&previous,
|
||||
&previous_state,
|
||||
disabled_revision,
|
||||
context,
|
||||
error,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
self.publish_instance(current, prepared, &result, replacements)
|
||||
.await?;
|
||||
drop(migration_prepared);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn recover_failed_state_transition(
|
||||
&self,
|
||||
transition_id: Option<&str>,
|
||||
previous: &PluginInstance,
|
||||
previous_state: &PluginStateConfiguration,
|
||||
disabled_revision: Option<Revision>,
|
||||
context: &MutationContext,
|
||||
original_error: AdminError,
|
||||
) -> AdminError {
|
||||
if let Some(transition_id) = transition_id {
|
||||
self.state.abort_transition(transition_id).await;
|
||||
}
|
||||
let Some(disabled_revision) = disabled_revision else {
|
||||
return original_error;
|
||||
};
|
||||
let restored = async {
|
||||
let mut snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
if snapshot.config_revision != disabled_revision {
|
||||
return Err(AdminError::conflict(
|
||||
"状态迁移失败;实例已保持停用,且配置已被其他操作修改",
|
||||
));
|
||||
}
|
||||
let candidate_revision = next_revision(disabled_revision)?;
|
||||
let mut restore = previous.clone();
|
||||
restore.enabled = true;
|
||||
restore.revision = candidate_revision;
|
||||
snapshot.instances.retain(|item| item.id != restore.id);
|
||||
snapshot.instances.push(restore.clone());
|
||||
snapshot.config_revision = candidate_revision;
|
||||
let prepared = self.preparation.prepare(snapshot.clone()).await?;
|
||||
let mutation = self
|
||||
.store
|
||||
.save_instance_with_state(
|
||||
restore,
|
||||
disabled_revision,
|
||||
PluginStateCommit {
|
||||
configuration: previous_state.clone(),
|
||||
transition_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
self.publish_instance(snapshot, prepared, &mutation, &[])
|
||||
.await?;
|
||||
Ok::<(), AdminError>(())
|
||||
}
|
||||
.await;
|
||||
match restored {
|
||||
Ok(()) => original_error,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
error_kind = ?error.kind(),
|
||||
"plugin state migration failed and the previous instance could not be restored"
|
||||
);
|
||||
AdminError::conflict("状态迁移失败;为避免覆盖并发配置,插件实例已保持停用")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! 插件实例用例的职责边界与静态检查结果
|
||||
|
||||
mod configuration;
|
||||
mod migration;
|
||||
mod mutations;
|
||||
mod publication;
|
||||
mod queries;
|
||||
|
||||
use super::PluginsService;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct PackageStatus {
|
||||
pub(super) warning: Option<String>,
|
||||
pub(super) load_error: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
//! 插件实例的配置提交、发布、停用与删除用例
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::{
|
||||
PluginsService,
|
||||
configuration::{apply_replacements, next_revision, same_creation_request, validate_input},
|
||||
};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
plugins::{
|
||||
instances::{
|
||||
ConfigurePluginInstance, PluginInstance, PluginInstanceMutation,
|
||||
PluginInstanceSnapshot,
|
||||
},
|
||||
state::PluginStateCommit,
|
||||
},
|
||||
},
|
||||
use_case::{map_store_error, publish_committed},
|
||||
};
|
||||
|
||||
impl PluginsService {
|
||||
pub async fn configure_instance(
|
||||
&self,
|
||||
id: Option<&str>,
|
||||
input: ConfigurePluginInstance,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
if id.is_some() && input.creation_id.is_some() {
|
||||
return Err(AdminError::invalid("编辑配置不能使用创建标识"));
|
||||
}
|
||||
if id.is_none() && input.expected_revision.is_some() {
|
||||
return Err(AdminError::invalid("创建配置不能指定已有配置版本"));
|
||||
}
|
||||
if let Some(creation_id) = &input.creation_id {
|
||||
let parsed = uuid::Uuid::parse_str(creation_id)
|
||||
.map_err(|_| AdminError::invalid("配置创建标识必须是 UUID"))?;
|
||||
if parsed.to_string() != *creation_id {
|
||||
return Err(AdminError::invalid("配置创建标识必须使用标准 UUID 格式"));
|
||||
}
|
||||
}
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let existing = match id {
|
||||
Some(id) => Some(
|
||||
snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.cloned()
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?,
|
||||
),
|
||||
None => input.creation_id.as_ref().and_then(|creation_id| {
|
||||
snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == *creation_id)
|
||||
.cloned()
|
||||
}),
|
||||
};
|
||||
if let Some(existing) = &existing {
|
||||
if id.is_none() && !same_creation_request(existing, &input) {
|
||||
return Err(AdminError::conflict("此配置已创建,请加载最新配置后编辑"));
|
||||
}
|
||||
if input
|
||||
.expected_revision
|
||||
.is_some_and(|revision| revision != existing.revision.get())
|
||||
{
|
||||
return Err(AdminError::conflict("插件配置已变更,请加载最新配置后重试"));
|
||||
}
|
||||
}
|
||||
self.configure_from_snapshot(snapshot, existing, input, context)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::use_case::plugins) async fn configure_from_snapshot(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
existing: Option<PluginInstance>,
|
||||
input: ConfigurePluginInstance,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let expected_revision = snapshot.config_revision;
|
||||
let candidate_revision = next_revision(expected_revision)?;
|
||||
let artifacts = self
|
||||
.store
|
||||
.list_artifacts()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
let artifact = artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.metadata.sha256 == input.artifact_sha256)
|
||||
.ok_or_else(|| AdminError::not_found("插件制品不存在"))?;
|
||||
if artifact.accepted_at.is_none() {
|
||||
return Err(AdminError::invalid("请先安装并确认完整信任该版本"));
|
||||
}
|
||||
if input.enabled
|
||||
&& let Some(warning) = self
|
||||
.package_status(&input.artifact_sha256)
|
||||
.await?
|
||||
.load_error
|
||||
{
|
||||
return Err(AdminError::invalid(format!(
|
||||
"{warning},无法启动,请安装兼容版本"
|
||||
)));
|
||||
}
|
||||
let same_plugin = |digest: &str| {
|
||||
artifacts.iter().any(|item| {
|
||||
item.metadata.sha256 == digest
|
||||
&& item.metadata.plugin_id == artifact.metadata.plugin_id
|
||||
})
|
||||
};
|
||||
if existing
|
||||
.as_ref()
|
||||
.is_some_and(|previous| !same_plugin(&previous.artifact_sha256))
|
||||
{
|
||||
return Err(AdminError::invalid("不能将已有配置切换为另一个插件"));
|
||||
}
|
||||
if input.enabled
|
||||
&& snapshot.instances.iter().any(|other| {
|
||||
other.enabled
|
||||
&& existing
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.id != other.id)
|
||||
&& same_plugin(&other.artifact_sha256)
|
||||
&& !input
|
||||
.replace_instances
|
||||
.iter()
|
||||
.any(|replacement| replacement.id == other.id)
|
||||
})
|
||||
{
|
||||
return Err(AdminError::conflict(
|
||||
"此插件已有启用配置,请先切换或停用旧配置",
|
||||
));
|
||||
}
|
||||
let saved = if let Some(previous) = &existing
|
||||
&& previous.artifact_sha256 != input.artifact_sha256
|
||||
&& input.secrets.is_none()
|
||||
{
|
||||
self.store
|
||||
.load_version_configuration(&previous.id, &input.artifact_sha256)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let instance = PluginInstance {
|
||||
id: existing.as_ref().map_or_else(
|
||||
|| {
|
||||
input
|
||||
.creation_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string())
|
||||
},
|
||||
|instance| instance.id.clone(),
|
||||
),
|
||||
name: input.name.trim().to_owned(),
|
||||
artifact_sha256: input.artifact_sha256,
|
||||
enabled: input.enabled,
|
||||
trusted_process: true,
|
||||
configuration: input.configuration,
|
||||
secrets: input
|
||||
.secrets
|
||||
.or_else(|| saved.map(|saved| saved.secrets))
|
||||
.unwrap_or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.map_or_else(Default::default, |instance| instance.secrets.clone())
|
||||
}),
|
||||
bindings: input.bindings,
|
||||
revision: candidate_revision,
|
||||
};
|
||||
validate_input(&instance)?;
|
||||
if existing
|
||||
.as_ref()
|
||||
.is_some_and(|previous| previous.artifact_sha256 != instance.artifact_sha256)
|
||||
&& !self
|
||||
.preparation
|
||||
.configuration_ready(instance.clone(), &artifact.metadata)
|
||||
.await?
|
||||
{
|
||||
return Err(AdminError::invalid(
|
||||
"目标版本仍缺少必填设置,请补充后再切换",
|
||||
));
|
||||
}
|
||||
|
||||
let replacements = input.replace_instances;
|
||||
if !replacements.is_empty() {
|
||||
if !instance.enabled || replacements.len() > 256 {
|
||||
return Err(AdminError::invalid(
|
||||
"仅启用配置时可以切换其他配置,单次最多 256 项",
|
||||
));
|
||||
}
|
||||
let mut ids = BTreeSet::new();
|
||||
for replacement in &replacements {
|
||||
if replacement.id == instance.id || !ids.insert(&replacement.id) {
|
||||
return Err(AdminError::invalid("停用列表不能重复或包含当前配置"));
|
||||
}
|
||||
let previous = snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|item| item.id == replacement.id)
|
||||
.ok_or_else(|| AdminError::conflict("待停用配置已变更,请重新确认"))?;
|
||||
if !previous.enabled || previous.revision.get() != replacement.expected_revision {
|
||||
return Err(AdminError::conflict("待停用配置已变更,请重新确认"));
|
||||
}
|
||||
if !artifacts.iter().any(|item| {
|
||||
item.metadata.sha256 == previous.artifact_sha256
|
||||
&& item.metadata.plugin_id == artifact.metadata.plugin_id
|
||||
}) {
|
||||
return Err(AdminError::invalid("只能切换同一插件的配置"));
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = self.preparation.validate(instance.clone()).await?;
|
||||
if let Some(existing) = &existing
|
||||
&& self.state.transition_required(&existing.id, &state).await?
|
||||
{
|
||||
return self
|
||||
.configure_instance_with_state_transition(
|
||||
snapshot,
|
||||
existing.clone(),
|
||||
instance,
|
||||
state,
|
||||
&replacements,
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
snapshot.instances.retain(|item| item.id != instance.id);
|
||||
snapshot.instances.push(instance.clone());
|
||||
snapshot.config_revision = candidate_revision;
|
||||
apply_replacements(&mut snapshot, &replacements);
|
||||
let prepared = self.preparation.prepare(snapshot.clone()).await?;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance_replacing(
|
||||
instance,
|
||||
expected_revision,
|
||||
PluginStateCommit {
|
||||
configuration: state,
|
||||
transition_id: None,
|
||||
},
|
||||
&replacements,
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
self.publish_instance(snapshot, prepared, &result, &replacements)
|
||||
.await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn delete_instance(
|
||||
&self,
|
||||
id: &str,
|
||||
context: &MutationContext,
|
||||
) -> Result<Revision, AdminError> {
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let instance = snapshot
|
||||
.instances
|
||||
.iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?;
|
||||
if instance.enabled {
|
||||
return Err(AdminError::conflict("请先停用插件实例"));
|
||||
}
|
||||
let revision = self
|
||||
.store
|
||||
.delete_instance(id, snapshot.config_revision, context)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), revision).await?;
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
/// 紧急管理修复不要求损坏插件成功准备;提交后沿用现有发布与暂停合同
|
||||
pub async fn disable_instance(
|
||||
&self,
|
||||
id: &str,
|
||||
context: &MutationContext,
|
||||
) -> Result<PluginInstanceMutation, AdminError> {
|
||||
let PluginInstanceSnapshot {
|
||||
config_revision,
|
||||
instances,
|
||||
} = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let mut instance = instances
|
||||
.into_iter()
|
||||
.find(|instance| instance.id == id)
|
||||
.ok_or_else(|| AdminError::not_found("插件实例不存在"))?;
|
||||
instance.enabled = false;
|
||||
let result = self
|
||||
.store
|
||||
.save_instance(instance, config_revision, context)
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
publish_committed(self.snapshots.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//! 配置提交与迁移恢复共用的状态激活和发布收尾
|
||||
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
|
||||
use super::{PluginsService, configuration::apply_replacements};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
plugins::instances::{
|
||||
PluginInstanceMutation, PluginInstanceReplacement, PluginInstanceSnapshot,
|
||||
},
|
||||
},
|
||||
use_case::publish_committed,
|
||||
};
|
||||
|
||||
impl PluginsService {
|
||||
/// 以事务返回的版本激活并发布,候选强引用必须覆盖整个提交后阶段
|
||||
pub(super) async fn publish_instance(
|
||||
&self,
|
||||
mut snapshot: PluginInstanceSnapshot,
|
||||
prepared: ExtensionSetReference,
|
||||
mutation: &PluginInstanceMutation,
|
||||
replacements: &[PluginInstanceReplacement],
|
||||
) -> Result<(), AdminError> {
|
||||
let prepared = if mutation.config_revision == snapshot.config_revision
|
||||
&& mutation.instance.revision == snapshot.config_revision
|
||||
{
|
||||
prepared
|
||||
} else {
|
||||
snapshot.config_revision = mutation.config_revision;
|
||||
snapshot
|
||||
.instances
|
||||
.retain(|item| item.id != mutation.instance.id);
|
||||
snapshot.instances.push(mutation.instance.clone());
|
||||
apply_replacements(&mut snapshot, replacements);
|
||||
self.preparation.prepare(snapshot).await?
|
||||
};
|
||||
self.preparation
|
||||
.activate_state(&prepared, &mutation.instance)
|
||||
.await?;
|
||||
publish_committed(self.snapshots.as_ref(), mutation.config_revision).await?;
|
||||
drop(prepared);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//! 插件实例列表与静态包检查的只读投影,不从发布事实推断进程状态
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::{PackageStatus, PluginsService};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind,
|
||||
plugins::instances::{
|
||||
PluginInstanceRuntime, PluginInstanceRuntimeStatus, PluginInstanceView,
|
||||
},
|
||||
},
|
||||
use_case::map_store_error,
|
||||
};
|
||||
|
||||
impl PluginsService {
|
||||
/// 只校验固定包体,不启动插件;停用实例也需区分兼容性提醒与加载错误
|
||||
pub(in crate::use_case::plugins) async fn package_status(
|
||||
&self,
|
||||
digest: &str,
|
||||
) -> Result<PackageStatus, AdminError> {
|
||||
// 运行宿主和摘要固定,缓存纯静态结论,避免管理页轮询反复解包
|
||||
// 只缓存有限数量的确定结果,暂时性错误仍可重试
|
||||
if let Some(status) = self.compatibility.lock().await.get(digest).cloned() {
|
||||
return Ok(status);
|
||||
}
|
||||
let artifact = self
|
||||
.store
|
||||
.load_artifact(digest)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin"))?;
|
||||
let status = match self
|
||||
.inspector
|
||||
.compatibility_warning(artifact.archive, digest.to_owned())
|
||||
.await
|
||||
{
|
||||
Ok(warning) => PackageStatus {
|
||||
warning,
|
||||
load_error: None,
|
||||
},
|
||||
Err(error) if error.kind() == AdminErrorKind::Invalid => PackageStatus {
|
||||
warning: None,
|
||||
load_error: Some(error.message().to_owned()),
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mut cache = self.compatibility.lock().await;
|
||||
if cache.len() < 128 {
|
||||
cache.insert(digest.to_owned(), status.clone());
|
||||
}
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub async fn instances(&self) -> Result<Vec<PluginInstanceView>, AdminError> {
|
||||
let snapshot = self
|
||||
.store
|
||||
.load_instances()
|
||||
.await
|
||||
.map_err(|e| map_store_error(e, "plugin"))?;
|
||||
let published = self.published.snapshot_for_diagnostics();
|
||||
let published_revision = published.as_ref().map(|snapshot| snapshot.revision().get());
|
||||
let mut diagnostics = self
|
||||
.preparation
|
||||
.runtime_diagnostics(
|
||||
&snapshot,
|
||||
published_revision,
|
||||
published
|
||||
.as_ref()
|
||||
.and_then(|snapshot| snapshot.extensions()),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let artifacts = self
|
||||
.list()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|artifact| (artifact.metadata.sha256.clone(), artifact.metadata))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut views = Vec::with_capacity(snapshot.instances.len());
|
||||
for instance in snapshot.instances {
|
||||
let metadata = artifacts
|
||||
.get(&instance.artifact_sha256)
|
||||
.ok_or_else(|| AdminError::not_found("插件制品不存在"))?;
|
||||
let package_status = self.package_status(&instance.artifact_sha256).await?;
|
||||
let configuration_required = package_status.load_error.is_none()
|
||||
&& !self
|
||||
.preparation
|
||||
.configuration_ready(instance.clone(), metadata)
|
||||
.await?;
|
||||
let runtime =
|
||||
diagnostics
|
||||
.remove(&instance.id)
|
||||
.unwrap_or_else(|| PluginInstanceRuntime {
|
||||
status: if !instance.enabled {
|
||||
PluginInstanceRuntimeStatus::Disabled
|
||||
} else {
|
||||
PluginInstanceRuntimeStatus::Unknown
|
||||
},
|
||||
actual_revision: None,
|
||||
actual_artifact_sha256: None,
|
||||
failure: None,
|
||||
draining_revisions: Vec::new(),
|
||||
load: None,
|
||||
});
|
||||
views.push(PluginInstanceView {
|
||||
api_deprecations: if package_status.load_error.is_none() {
|
||||
self.inspector.api_deprecations(metadata)?
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
configuration_required,
|
||||
compatibility_warning: package_status.warning,
|
||||
load_error: package_status.load_error,
|
||||
running: runtime.status == PluginInstanceRuntimeStatus::Running,
|
||||
published_revision,
|
||||
runtime,
|
||||
instance,
|
||||
});
|
||||
}
|
||||
Ok(views)
|
||||
}
|
||||
}
|
||||
+15
-43
@@ -49,6 +49,8 @@ use serde_json::json;
|
||||
|
||||
use super::TestPluginPorts;
|
||||
|
||||
mod queries;
|
||||
|
||||
const OLD_ARTIFACT: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
const NEW_ARTIFACT: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
|
||||
|
||||
@@ -88,6 +90,11 @@ impl gateway_admin::ports::plugins::PluginPackageInspector for LifecycleFixture
|
||||
archive: Arc<[u8]>,
|
||||
digest: String,
|
||||
) -> Result<Option<String>, AdminError> {
|
||||
let gate = self.inspection_gate.lock().unwrap().clone();
|
||||
if let Some(gate) = gate.filter(|gate| gate.digest == digest) {
|
||||
gate.started.notify_one();
|
||||
gate.release.notified().await;
|
||||
}
|
||||
self.inspect(archive, Some(digest)).await?;
|
||||
Ok(self.data.lock().unwrap().compatibility_warning.clone())
|
||||
}
|
||||
@@ -143,11 +150,19 @@ struct LifecycleFixture {
|
||||
behavior: MigrationBehavior,
|
||||
data: Mutex<FixtureData>,
|
||||
next_prepared: AtomicU64,
|
||||
inspection_gate: Mutex<Option<Arc<InspectionGate>>>,
|
||||
}
|
||||
|
||||
struct InspectionGate {
|
||||
digest: String,
|
||||
started: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl LifecycleFixture {
|
||||
fn new(behavior: MigrationBehavior) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
inspection_gate: Mutex::new(None),
|
||||
behavior,
|
||||
data: Mutex::new(FixtureData {
|
||||
snapshot: PluginInstanceSnapshot {
|
||||
@@ -970,49 +985,6 @@ fn state_unavailable<T>() -> PluginStateStoreResult<T> {
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn instance_list_uses_runtime_diagnostics_and_keeps_running_compatibility_derived() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let expected = PluginInstanceRuntime {
|
||||
status: PluginInstanceRuntimeStatus::PreparationFailed,
|
||||
actual_revision: Some(9),
|
||||
actual_artifact_sha256: Some(OLD_ARTIFACT.into()),
|
||||
failure: Some(PluginInstanceRuntimeFailure {
|
||||
code: "unavailable".into(),
|
||||
message: "插件候选准备失败".into(),
|
||||
}),
|
||||
draining_revisions: vec![8],
|
||||
};
|
||||
fixture.data.lock().unwrap().diagnostics = Some(BTreeMap::from([(
|
||||
"00000000-0000-7000-8000-000000000001".into(),
|
||||
expected.clone(),
|
||||
)]));
|
||||
|
||||
let views = service(fixture, Arc::new(Published::default()))
|
||||
.instances()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(views.len(), 1);
|
||||
assert!(!views[0].running);
|
||||
assert_eq!(views[0].runtime, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn instance_list_falls_back_without_inventing_runtime_failures() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let views = service(fixture, Arc::new(Published::default()))
|
||||
.instances()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(views.len(), 1);
|
||||
assert_eq!(
|
||||
views[0].runtime.status,
|
||||
PluginInstanceRuntimeStatus::AwaitingPublication
|
||||
);
|
||||
assert!(!views[0].running);
|
||||
assert!(views[0].runtime.failure.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incompatible_state_upgrade_quiesces_migrates_and_promotes_the_target() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
@@ -0,0 +1,151 @@
|
||||
//! 实例运行诊断与包检查缓存的查询回归测试
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::model::plugins::instances::PluginInstanceRuntimeLoad;
|
||||
|
||||
#[tokio::test]
|
||||
async fn instance_list_uses_runtime_diagnostics_and_keeps_running_compatibility_derived() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let expected = PluginInstanceRuntime {
|
||||
status: PluginInstanceRuntimeStatus::PreparationFailed,
|
||||
actual_revision: Some(9),
|
||||
actual_artifact_sha256: Some(OLD_ARTIFACT.into()),
|
||||
failure: Some(PluginInstanceRuntimeFailure {
|
||||
code: "unavailable".into(),
|
||||
message: "插件候选准备失败".into(),
|
||||
}),
|
||||
draining_revisions: vec![8],
|
||||
load: Some(PluginInstanceRuntimeLoad {
|
||||
active_calls: 3,
|
||||
executing_calls: 1,
|
||||
queued_calls: 1,
|
||||
resuming_calls: 1,
|
||||
pending_callbacks: 2,
|
||||
active_streams: 0,
|
||||
}),
|
||||
};
|
||||
fixture.data.lock().unwrap().diagnostics = Some(BTreeMap::from([(
|
||||
"00000000-0000-7000-8000-000000000001".into(),
|
||||
expected.clone(),
|
||||
)]));
|
||||
|
||||
let views = service(fixture, Arc::new(Published::default()))
|
||||
.instances()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(views.len(), 1);
|
||||
assert!(!views[0].running);
|
||||
assert_eq!(views[0].runtime, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn instance_list_falls_back_without_inventing_runtime_failures() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let views = service(fixture, Arc::new(Published::default()))
|
||||
.instances()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(views.len(), 1);
|
||||
assert_eq!(
|
||||
views[0].runtime.status,
|
||||
PluginInstanceRuntimeStatus::Unknown
|
||||
);
|
||||
assert!(!views[0].running);
|
||||
assert!(views[0].runtime.failure.is_none());
|
||||
assert!(views[0].runtime.actual_revision.is_none());
|
||||
assert!(views[0].runtime.load.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_instance_without_runtime_diagnostics_has_a_known_disabled_status() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].enabled = false;
|
||||
let views = service(fixture, Arc::new(Published::default()))
|
||||
.instances()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
views[0].runtime.status,
|
||||
PluginInstanceRuntimeStatus::Disabled
|
||||
);
|
||||
assert!(!views[0].running);
|
||||
assert!(views[0].runtime.failure.is_none());
|
||||
assert!(views[0].runtime.load.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn matching_publication_without_process_diagnostics_does_not_imply_running() {
|
||||
for diagnostics in [None, Some(BTreeMap::new())] {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
fixture.data.lock().unwrap().diagnostics = diagnostics;
|
||||
let revision = fixture.snapshot().config_revision.get();
|
||||
let published = RuntimeSnapshotHandle::new(
|
||||
gateway_core::routing::RuntimeSnapshot::new(
|
||||
ConfigRevision::new(revision).unwrap(),
|
||||
gateway_core::settings::SettingsValues::new(
|
||||
1,
|
||||
"round_robin",
|
||||
BTreeMap::new(),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let service = PluginsService::new(
|
||||
fixture.clone(),
|
||||
fixture.clone(),
|
||||
PluginDistributionPorts::new(Arc::new(TestPluginPorts), Arc::new(TestPluginPorts)),
|
||||
Arc::new(Published::default()),
|
||||
fixture.clone(),
|
||||
published,
|
||||
fixture,
|
||||
);
|
||||
let view = service.instances().await.unwrap().remove(0);
|
||||
assert_eq!(view.published_revision, Some(revision));
|
||||
assert_eq!(view.runtime.status, PluginInstanceRuntimeStatus::Unknown);
|
||||
assert!(!view.running);
|
||||
assert!(view.runtime.actual_revision.is_none());
|
||||
assert!(view.runtime.actual_artifact_sha256.is_none());
|
||||
assert!(view.runtime.failure.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_package_query_is_not_blocked_by_another_package_inspection() {
|
||||
let fixture = LifecycleFixture::new(MigrationBehavior::Succeed);
|
||||
let service = Arc::new(service(fixture.clone(), Arc::new(Published::default())));
|
||||
service.instances().await.unwrap();
|
||||
let gate = Arc::new(InspectionGate {
|
||||
digest: NEW_ARTIFACT.into(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*fixture.inspection_gate.lock().unwrap() = Some(gate.clone());
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].artifact_sha256 = NEW_ARTIFACT.into();
|
||||
let inspecting = {
|
||||
let service = service.clone();
|
||||
tokio::spawn(async move { service.instances().await })
|
||||
};
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), gate.started.notified())
|
||||
.await
|
||||
.expect("another package inspection started");
|
||||
|
||||
fixture.data.lock().unwrap().snapshot.instances[0].artifact_sha256 = OLD_ARTIFACT.into();
|
||||
let cached =
|
||||
tokio::time::timeout(std::time::Duration::from_millis(100), service.instances()).await;
|
||||
gate.release.notify_one();
|
||||
inspecting.await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
cached
|
||||
.expect("cached query must not wait for inspection")
|
||||
.unwrap()[0]
|
||||
.instance
|
||||
.artifact_sha256,
|
||||
OLD_ARTIFACT
|
||||
);
|
||||
assert_eq!(fixture.data.lock().unwrap().archive_reads, 2);
|
||||
}
|
||||
@@ -9,8 +9,8 @@ use axum::{
|
||||
};
|
||||
use gateway_admin::model::plugins::instances::{
|
||||
ConfigurePluginInstance, PluginCapabilityBinding, PluginInstance, PluginInstanceRuntime,
|
||||
PluginInstanceRuntimeFailure, PluginInstanceRuntimeStatus, PluginInstanceView,
|
||||
RollbackPluginInstance,
|
||||
PluginInstanceRuntimeFailure, PluginInstanceRuntimeLoad, PluginInstanceRuntimeStatus,
|
||||
PluginInstanceView, RollbackPluginInstance,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -70,6 +70,7 @@ struct InstanceView {
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum InstanceRuntimeStatus {
|
||||
Disabled,
|
||||
Unknown,
|
||||
AwaitingPublication,
|
||||
Preparing,
|
||||
Running,
|
||||
@@ -83,6 +84,7 @@ impl From<PluginInstanceRuntimeStatus> for InstanceRuntimeStatus {
|
||||
fn from(value: PluginInstanceRuntimeStatus) -> Self {
|
||||
match value {
|
||||
PluginInstanceRuntimeStatus::Disabled => Self::Disabled,
|
||||
PluginInstanceRuntimeStatus::Unknown => Self::Unknown,
|
||||
PluginInstanceRuntimeStatus::AwaitingPublication => Self::AwaitingPublication,
|
||||
PluginInstanceRuntimeStatus::Preparing => Self::Preparing,
|
||||
PluginInstanceRuntimeStatus::Running => Self::Running,
|
||||
@@ -118,6 +120,31 @@ struct InstanceRuntimeView {
|
||||
actual_artifact_sha256: Option<String>,
|
||||
failure: Option<InstanceRuntimeFailure>,
|
||||
draining_revisions: Vec<u64>,
|
||||
load: Option<InstanceRuntimeLoad>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct InstanceRuntimeLoad {
|
||||
active_calls: usize,
|
||||
executing_calls: usize,
|
||||
queued_calls: usize,
|
||||
resuming_calls: usize,
|
||||
pending_callbacks: usize,
|
||||
active_streams: usize,
|
||||
}
|
||||
|
||||
impl From<PluginInstanceRuntimeLoad> for InstanceRuntimeLoad {
|
||||
fn from(value: PluginInstanceRuntimeLoad) -> Self {
|
||||
Self {
|
||||
active_calls: value.active_calls,
|
||||
executing_calls: value.executing_calls,
|
||||
queued_calls: value.queued_calls,
|
||||
resuming_calls: value.resuming_calls,
|
||||
pending_callbacks: value.pending_callbacks,
|
||||
active_streams: value.active_streams,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<PluginInstanceRuntime> for InstanceRuntimeView {
|
||||
@@ -128,6 +155,7 @@ impl From<PluginInstanceRuntime> for InstanceRuntimeView {
|
||||
actual_artifact_sha256: value.actual_artifact_sha256,
|
||||
failure: value.failure.map(Into::into),
|
||||
draining_revisions: value.draining_revisions,
|
||||
load: value.load.map(Into::into),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -322,6 +322,17 @@ async fn instance_list_serializes_safe_runtime_diagnostics() {
|
||||
assert_eq!(instance["running"], false);
|
||||
assert_eq!(instance["runtime"]["status"], "preparation_failed");
|
||||
assert_eq!(instance["runtime"]["actualRevision"], 6);
|
||||
assert_eq!(
|
||||
instance["runtime"]["load"],
|
||||
serde_json::json!({
|
||||
"activeCalls": 3,
|
||||
"executingCalls": 1,
|
||||
"queuedCalls": 1,
|
||||
"resumingCalls": 1,
|
||||
"pendingCallbacks": 2,
|
||||
"activeStreams": 0,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
instance["runtime"]["drainingRevisions"],
|
||||
serde_json::json!([5])
|
||||
@@ -665,6 +676,14 @@ impl gateway_admin::ports::plugins::PluginRuntimeDiagnostics for TestPluginPorts
|
||||
message: "插件候选准备失败".into(),
|
||||
}),
|
||||
draining_revisions: vec![5],
|
||||
load: Some(gateway_admin::model::plugins::instances::PluginInstanceRuntimeLoad {
|
||||
active_calls: 3,
|
||||
executing_calls: 1,
|
||||
queued_calls: 1,
|
||||
resuming_calls: 1,
|
||||
pending_callbacks: 2,
|
||||
active_streams: 0,
|
||||
}),
|
||||
},
|
||||
)
|
||||
})
|
||||
|
||||
@@ -2164,109 +2164,132 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_upstream_error_event_should_be_translated_to_response_failed_for_sse() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error_data = json!({
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "service_unavailable_error",
|
||||
"code": "server_is_overloaded",
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
});
|
||||
let projected_error_frame = Bytes::from(encode_sse_event("error", &error_data.to_string()));
|
||||
let error_wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("error".to_owned()),
|
||||
error_data,
|
||||
projected_error_frame,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream error wire"),
|
||||
);
|
||||
let started_wire = ProviderEvent::canonical_with_wire(
|
||||
vec![started()],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "in_progress",
|
||||
"future_response_field": {"keep": true}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("valid upstream started wire"),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(CoordinatedEvent::single(
|
||||
started_wire,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::Event(CoordinatedEvent::single(
|
||||
error_wire,
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)),
|
||||
NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
UpstreamSendState::Sent,
|
||||
))),
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("SSE is UTF-8");
|
||||
let events = parse_sse_events(&body).expect("translated SSE should parse");
|
||||
let failed = events
|
||||
.iter()
|
||||
.find(|event| event.event.as_deref() == Some("response.failed"))
|
||||
.expect("translated response.failed event");
|
||||
let failed: Value = serde_json::from_str(&failed.data).expect("response.failed JSON");
|
||||
|
||||
assert_eq!(
|
||||
for (error_type, code, client_code, kind, client_status) in [
|
||||
(
|
||||
failed,
|
||||
body.matches("event: response.failed").count(),
|
||||
body.contains("event: error\n"),
|
||||
body.ends_with("data: [DONE]\n\n"),
|
||||
"service_unavailable_error",
|
||||
"server_is_overloaded",
|
||||
"server_error",
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
503,
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": "service_unavailable_error",
|
||||
"code": "server_error",
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true}
|
||||
"rate_limit_error",
|
||||
"turn_rate_limit_exceeded",
|
||||
"turn_rate_limit_exceeded",
|
||||
ProviderErrorKind::RateLimited,
|
||||
429,
|
||||
),
|
||||
] {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error_data = json!({
|
||||
"type": "error",
|
||||
"status": 429,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
});
|
||||
let projected_error_frame = Bytes::from(encode_sse_event("error", &error_data.to_string()));
|
||||
let error_wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("error".to_owned()),
|
||||
error_data,
|
||||
projected_error_frame,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream error wire"),
|
||||
);
|
||||
let started_wire = ProviderEvent::canonical_with_wire(
|
||||
vec![started()],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "in_progress",
|
||||
"future_response_field": {"keep": true}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("valid upstream started wire"),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(CoordinatedEvent::single(
|
||||
started_wire,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::Event(CoordinatedEvent::single(
|
||||
error_wire,
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)),
|
||||
NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
kind,
|
||||
UpstreamSendState::Sent,
|
||||
))),
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("SSE is UTF-8");
|
||||
let events = parse_sse_events(&body).expect("translated SSE should parse");
|
||||
let failed = events
|
||||
.iter()
|
||||
.find(|event| event.event.as_deref() == Some("response.failed"))
|
||||
.expect("translated response.failed event");
|
||||
let failed: Value = serde_json::from_str(&failed.data).expect("response.failed JSON");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
failed,
|
||||
body.matches("event: response.failed").count(),
|
||||
body.contains("event: error\n"),
|
||||
body.ends_with("data: [DONE]\n\n"),
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"status": client_status,
|
||||
"headers": {"Retry-After": "24"},
|
||||
"response": {
|
||||
"id": "resp_test",
|
||||
"model": "public-model",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": client_code,
|
||||
"message": "Our servers are currently overloaded. Please try again later.",
|
||||
"param": null,
|
||||
"future_error_field": {"keep": true},
|
||||
"headers": {"retry-after": "24"}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"future_response_field": {"keep": true}
|
||||
},
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
}),
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
);
|
||||
"sequence_number": 2,
|
||||
"future_event_field": {"keep": true}
|
||||
}),
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -948,8 +948,8 @@ fn transparent_encoder_should_translate_error_wire_to_response_failed_for_websoc
|
||||
json!({
|
||||
"type": "error",
|
||||
"status": 429,
|
||||
"headers": {"x-request-id": "req_raw"},
|
||||
"error": {"type": "custom_type", "code": "custom_code", "message": "raw marker"},
|
||||
"headers": {"x-request-id": "req_raw", "Retry-After": "24"},
|
||||
"error": {"type": "rate_limit_error", "code": "turn_rate_limit_exceeded", "message": "raw marker"},
|
||||
"future": {"keep": true}
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -9,15 +9,38 @@ use std::{
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::{
|
||||
error::GatewayError,
|
||||
policy::{ClientApiKeyId, PlaintextClientApiKey},
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
routing::{
|
||||
extensions::{ExtensionSetId, ExtensionSetReference},
|
||||
request_settings::RequestSettings,
|
||||
},
|
||||
};
|
||||
|
||||
use super::execution::AuthenticatedClient;
|
||||
|
||||
/// 身份解析与认证只返回冻结权限,不提供模型执行入口
|
||||
pub trait ClientIdentityPort: Send + Sync {
|
||||
/// 只解析身份与冻结权限,不建立模型执行图或记录使用事实
|
||||
fn resolve_client(
|
||||
&self,
|
||||
client_key_id: ClientApiKeyId,
|
||||
settings: Option<RequestSettings>,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, GatewayError>>;
|
||||
|
||||
/// 复用入口认证计划及 principal 映射,不进行模型准入
|
||||
fn verify_client(
|
||||
&self,
|
||||
request: ClientAuthenticationRequest,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, GatewayError>>;
|
||||
}
|
||||
|
||||
pub const MAXIMUM_AUTHORIZATION_BYTES: usize = 8 * 1024;
|
||||
|
||||
/// API 从数据面 Authorization 头构造的有界认证信封
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
#[derive(Clone)]
|
||||
pub struct ClientAuthenticationRequest {
|
||||
invocation: Option<Arc<super::extensions::InvocationOrigin>>,
|
||||
request_context: Option<Arc<super::extensions::RequestContext>>,
|
||||
settings: Option<crate::routing::request_settings::RequestSettings>,
|
||||
authorization: Arc<str>,
|
||||
@@ -25,6 +48,18 @@ pub struct ClientAuthenticationRequest {
|
||||
}
|
||||
|
||||
impl ClientAuthenticationRequest {
|
||||
/// 子认证继承受信调用的来源、取消和副作用归属,不据此跳过认证器
|
||||
#[must_use]
|
||||
pub fn with_invocation(mut self, origin: super::extensions::InvocationOrigin) -> Self {
|
||||
self.invocation = Some(Arc::new(origin));
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn invocation(&self) -> Option<&super::extensions::InvocationOrigin> {
|
||||
self.invocation.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_context(
|
||||
mut self,
|
||||
@@ -35,12 +70,18 @@ impl ClientAuthenticationRequest {
|
||||
}
|
||||
#[must_use]
|
||||
pub fn request_context(&self) -> Option<&super::extensions::RequestContext> {
|
||||
self.request_context.as_deref()
|
||||
self.invocation
|
||||
.as_ref()
|
||||
.and_then(|origin| origin.extensions.request())
|
||||
.or(self.request_context.as_deref())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn shared_request_context(&self) -> Option<Arc<super::extensions::RequestContext>> {
|
||||
self.request_context.clone()
|
||||
self.invocation
|
||||
.as_ref()
|
||||
.and_then(|origin| origin.extensions.shared_request())
|
||||
.or_else(|| self.request_context.clone())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -79,6 +120,7 @@ impl ClientAuthenticationRequest {
|
||||
.filter(|value| !value.is_empty())
|
||||
.and_then(|value| PlaintextClientApiKey::new(value.to_owned()).ok());
|
||||
Ok(Self {
|
||||
invocation: None,
|
||||
settings: None,
|
||||
request_context: None,
|
||||
authorization: authorization.into(),
|
||||
|
||||
@@ -12,7 +12,6 @@ pub use contract::{
|
||||
StartedExecution,
|
||||
};
|
||||
mod scope;
|
||||
pub use scope::BoundModelExecutionContext;
|
||||
mod service;
|
||||
pub use failure::gateway_error_from_engine;
|
||||
pub use service::DefaultExecutionService;
|
||||
|
||||
@@ -15,13 +15,10 @@ use crate::{
|
||||
lifecycle::{CancellationToken, Deadline},
|
||||
operation::Operation,
|
||||
policy::ClientApiKeyId,
|
||||
routing::{
|
||||
FrozenAccountScope, PublicModelId, UpstreamModelId, request_settings::RequestSettings,
|
||||
},
|
||||
routing::{FrozenAccountScope, PublicModelId, UpstreamModelId},
|
||||
};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fmt,
|
||||
sync::{
|
||||
Arc, Mutex, Weak,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -188,57 +185,6 @@ pub(super) struct ActiveRequestLease {
|
||||
pub(super) authority: Arc<ActiveRequestAuthority>,
|
||||
}
|
||||
|
||||
/// 管理页或 CLI 一次调用持有的显式模型执行身份
|
||||
///
|
||||
/// 结构不暴露 Key 明文;Runtime 只能把它原样交回 Core 发起模型请求
|
||||
#[derive(Clone)]
|
||||
pub struct BoundModelExecutionContext {
|
||||
pub(super) authority: Arc<BoundModelExecutionAuthority>,
|
||||
}
|
||||
|
||||
impl BoundModelExecutionContext {
|
||||
/// 与实际执行共用绑定时的快照,不能回退到后续发布的配置
|
||||
#[must_use]
|
||||
pub fn request_body_limit(&self) -> usize {
|
||||
self.authority
|
||||
.client
|
||||
.snapshot
|
||||
.responses_max_decompressed_body_bytes()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn request_settings(&self) -> Option<&RequestSettings> {
|
||||
self.authority.client.request_settings()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct BoundModelExecutionAuthority {
|
||||
pub(super) client: AuthenticatedClient,
|
||||
pub(super) account_scope: Arc<FrozenAccountScope>,
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) extension_scope: ExtensionCallScope,
|
||||
pub(super) initiating_plugin_instance_id: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BoundModelExecutionContext {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BoundModelExecutionContext")
|
||||
.field("key_id", &self.authority.client.policy.key_id())
|
||||
.field(
|
||||
"initiating_plugin_instance_id",
|
||||
&self.authority.initiating_plugin_instance_id,
|
||||
)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BoundModelExecutionAuthority {
|
||||
fn drop(&mut self) {
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl ActiveRequestLease {
|
||||
pub(super) fn release(self) {
|
||||
drop(self);
|
||||
|
||||
@@ -12,9 +12,8 @@ use super::contract::{
|
||||
use super::probe_store::{ProbeObservation, TransientExecutionStore};
|
||||
use super::scope::{
|
||||
ActiveRequestAuthority, ActiveRequestLease, ActiveRequestRegistry, AuthorizedExecution,
|
||||
BoundExecutionFacts, BoundModelExecutionAuthority, BoundModelExecutionContext,
|
||||
ExecutionStartGuard, ExecutionTarget, NestedExecutionFacts, NestedExecutionGraph,
|
||||
PendingStartExecution, PreparedExecutionStart,
|
||||
BoundExecutionFacts, ExecutionStartGuard, ExecutionTarget, NestedExecutionFacts,
|
||||
NestedExecutionGraph, PendingStartExecution, PreparedExecutionStart,
|
||||
};
|
||||
use super::{COORDINATION_TIMEOUT, DIAGNOSTIC_TIMEOUT};
|
||||
use super::{scope::ExecutionAdmission, session::DefaultExecutionSession};
|
||||
@@ -28,7 +27,7 @@ use crate::{
|
||||
ClientAdmissionRequest,
|
||||
},
|
||||
authentication::{
|
||||
ClientAuthenticationRequest, FrontendAuthenticationDecision,
|
||||
ClientAuthenticationRequest, ClientIdentityPort, FrontendAuthenticationDecision,
|
||||
FrontendAuthenticationExtensionIndex,
|
||||
},
|
||||
budget::{ClientBudgetCharge, ClientBudgetPort},
|
||||
@@ -41,8 +40,8 @@ use crate::{
|
||||
middleware::FrozenMiddlewarePlan,
|
||||
nested::{
|
||||
AffinityLookupPort, AffinityLookupRequest, AffinityLookupResult,
|
||||
BoundModelExecutionBinding, BoundModelExecutionRequest, ExecutionEffects,
|
||||
NestedModelExecutionPort, NestedModelExecutionRequest,
|
||||
BoundModelExecutionRequest, ExecutionEffects, NestedModelExecutionPort,
|
||||
NestedModelExecutionRequest,
|
||||
},
|
||||
observation::{
|
||||
FrozenRequestObservationContext, RequestObservationDispatch, RequestObservationScope,
|
||||
@@ -1081,72 +1080,54 @@ impl DefaultExecutionService {
|
||||
})
|
||||
}
|
||||
|
||||
async fn bind_bound_model_inner(
|
||||
&self,
|
||||
binding: BoundModelExecutionBinding,
|
||||
) -> Result<BoundModelExecutionContext, GatewayError> {
|
||||
let client = self
|
||||
.client_for_key_id(&binding.client_key_id, binding.settings)
|
||||
.await?;
|
||||
if binding.extension_scope.len() >= ExtensionCallScope::MAXIMUM_DEPTH {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"plugin model execution depth exceeded",
|
||||
));
|
||||
}
|
||||
let extension_scope = binding
|
||||
.extension_scope
|
||||
.extending(binding.initiating_plugin_instance_id.clone())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"bound plugin execution scope is invalid",
|
||||
)
|
||||
})?;
|
||||
let account_scope = Arc::clone(client.policy.account_scope());
|
||||
Ok(BoundModelExecutionContext {
|
||||
authority: Arc::new(BoundModelExecutionAuthority {
|
||||
client,
|
||||
account_scope,
|
||||
cancellation: binding.cancellation,
|
||||
extension_scope,
|
||||
initiating_plugin_instance_id: binding.initiating_plugin_instance_id,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
async fn start_bound_model_inner(
|
||||
&self,
|
||||
request: BoundModelExecutionRequest,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
let BoundModelExecutionRequest {
|
||||
context,
|
||||
client,
|
||||
initiating_plugin_instance_id,
|
||||
cancellation,
|
||||
extension_scope,
|
||||
public_model,
|
||||
operation,
|
||||
mut metadata,
|
||||
provider,
|
||||
account,
|
||||
} = request;
|
||||
let authority = &context.authority;
|
||||
if authority.cancellation.is_cancelled() {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Cancelled,
|
||||
"bound plugin invocation was cancelled",
|
||||
));
|
||||
}
|
||||
let now = SystemTime::now();
|
||||
let deadline_at = Deadline::from_timeout(now, authority.client.execution_timeout())
|
||||
if extension_scope.len() >= ExtensionCallScope::MAXIMUM_DEPTH {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"plugin model execution depth exceeded",
|
||||
));
|
||||
}
|
||||
let extension_scope = extension_scope
|
||||
.extending(initiating_plugin_instance_id.clone())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"plugin model execution scope is recursive",
|
||||
)
|
||||
})?;
|
||||
let now = SystemTime::now();
|
||||
let deadline_at =
|
||||
Deadline::from_timeout(now, client.execution_timeout()).ok_or_else(|| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "system clock is invalid")
|
||||
})?;
|
||||
let (account_scope, required_provider) = restrict_model_execution_scope(
|
||||
authority.account_scope.as_ref(),
|
||||
client.policy.account_scope().as_ref(),
|
||||
provider,
|
||||
account.as_ref(),
|
||||
None,
|
||||
)?;
|
||||
// 连接承载多个独立执行;递归预算和副作用只在本次执行的子调用图内共享
|
||||
// 父连接的取消仍通过 authority 传播,不能因新建执行丢失扩展调用链
|
||||
// 父调用的取消继续向子执行传播,不能因新建执行丢失扩展调用链
|
||||
let graph = Arc::new(NestedExecutionGraph::new(Arc::new(
|
||||
ExecutionEffects::default(),
|
||||
)));
|
||||
@@ -1154,12 +1135,12 @@ impl DefaultExecutionService {
|
||||
let execution_effects_baseline = graph.effects.epoch();
|
||||
metadata.endpoint = "host.model".to_owned();
|
||||
metadata.transport = ClientTransport::InternalPlugin;
|
||||
metadata.inherit_request_origin(&authority.extension_scope);
|
||||
metadata.inherit_request_origin(&extension_scope);
|
||||
self.client_api_key_usage
|
||||
.record_used(authority.client.policy.key_id());
|
||||
.record_used(client.policy.key_id());
|
||||
self.start_authorized(
|
||||
PendingStartExecution {
|
||||
client: authority.client.clone(),
|
||||
client,
|
||||
target: ExecutionTarget::Model(public_model),
|
||||
operation,
|
||||
metadata,
|
||||
@@ -1168,13 +1149,13 @@ impl DefaultExecutionService {
|
||||
account_scope,
|
||||
response_control: None,
|
||||
deadline_at,
|
||||
cancellation: authority.cancellation.child_token(),
|
||||
extension_scope: authority.extension_scope.clone(),
|
||||
cancellation: cancellation.child_token(),
|
||||
extension_scope,
|
||||
required_provider,
|
||||
required_account: account,
|
||||
nested: None,
|
||||
bound: Some(BoundExecutionFacts {
|
||||
initiating_plugin_instance_id: authority.initiating_plugin_instance_id.clone(),
|
||||
initiating_plugin_instance_id,
|
||||
}),
|
||||
graph: Some(graph),
|
||||
execution_effects_baseline,
|
||||
@@ -1724,23 +1705,57 @@ impl ExecutionService for DefaultExecutionService {
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientIdentityPort for DefaultExecutionService {
|
||||
fn resolve_client(
|
||||
&self,
|
||||
client_key_id: ClientApiKeyId,
|
||||
settings: Option<RequestSettings>,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, GatewayError>> {
|
||||
Box::pin(async move { self.client_for_key_id(&client_key_id, settings).await })
|
||||
}
|
||||
|
||||
fn verify_client(
|
||||
&self,
|
||||
request: ClientAuthenticationRequest,
|
||||
) -> BoxFuture<'_, Result<AuthenticatedClient, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let cancellation = request
|
||||
.invocation()
|
||||
.map(|origin| origin.cancellation.clone());
|
||||
let verification = self.verify_request(request).fuse();
|
||||
pin_mut!(verification);
|
||||
match cancellation {
|
||||
Some(cancellation) => {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
pin_mut!(cancelled);
|
||||
select_biased! {
|
||||
() = cancelled => Err(GatewayError::new(GatewayErrorKind::Cancelled, "client verification was cancelled")),
|
||||
result = verification => result.map_err(authentication_gateway_error),
|
||||
}
|
||||
}
|
||||
None => verification.await.map_err(authentication_gateway_error),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl NestedModelExecutionPort for DefaultExecutionService {
|
||||
fn models(
|
||||
&self,
|
||||
context: BoundModelExecutionContext,
|
||||
client: AuthenticatedClient,
|
||||
cancellation: CancellationToken,
|
||||
protocol: String,
|
||||
client_version: String,
|
||||
) -> BoxFuture<'_, Result<Vec<PublicModelId>, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let authority = &context.authority;
|
||||
let cancellation = authority.cancellation.cancelled().fuse();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let timeout = Delay::new(DIAGNOSTIC_TIMEOUT).fuse();
|
||||
let catalog = self
|
||||
.client_model_catalog(&authority.client, &protocol, &client_version)
|
||||
.client_model_catalog(&client, &protocol, &client_version)
|
||||
.fuse();
|
||||
pin_mut!(cancellation, timeout, catalog);
|
||||
pin_mut!(cancelled, timeout, catalog);
|
||||
let models = select_biased! {
|
||||
() = cancellation => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "model catalog call was cancelled")),
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "model catalog call was cancelled")),
|
||||
() = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "model catalog deadline elapsed")),
|
||||
result = catalog => result.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "model catalog is unavailable"))?,
|
||||
};
|
||||
@@ -1754,13 +1769,6 @@ impl NestedModelExecutionPort for DefaultExecutionService {
|
||||
})
|
||||
}
|
||||
|
||||
fn bind(
|
||||
&self,
|
||||
binding: BoundModelExecutionBinding,
|
||||
) -> BoxFuture<'_, Result<BoundModelExecutionContext, GatewayError>> {
|
||||
Box::pin(async move { self.bind_bound_model_inner(binding).await })
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: NestedModelExecutionRequest,
|
||||
|
||||
@@ -62,6 +62,17 @@ impl ExtensionCallScope {
|
||||
}
|
||||
}
|
||||
|
||||
/// 宿主签发的父调用来源;身份裁决、执行链和实际资源仍由各自 owner 持有
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct InvocationOrigin {
|
||||
pub request_id: String,
|
||||
pub call_id: String,
|
||||
pub cancellation: crate::lifecycle::CancellationToken,
|
||||
pub extensions: ExtensionCallScope,
|
||||
pub plan: Option<super::middleware::FrozenMiddlewarePlan>,
|
||||
pub effects: Option<Arc<super::nested::ExecutionEffects>>,
|
||||
}
|
||||
|
||||
/// 同一代次的具体执行计划;不拥有发布状态,运行资源由对应发布集合保活
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ExecutionExtensionPlans {
|
||||
|
||||
@@ -13,13 +13,13 @@ use futures::future::BoxFuture;
|
||||
|
||||
use crate::{
|
||||
account::ProviderAccountId, error::GatewayError, identity::ProviderKind,
|
||||
lifecycle::CancellationToken, operation::Operation, policy::ClientApiKeyId,
|
||||
provider_ports::ProviderSessionAffinityKey, routing::PublicModelId,
|
||||
lifecycle::CancellationToken, operation::Operation, provider_ports::ProviderSessionAffinityKey,
|
||||
routing::PublicModelId,
|
||||
};
|
||||
|
||||
use super::{
|
||||
ModelRequestId,
|
||||
execution::{BoundModelExecutionContext, ExecutionRequestMetadata, StartedExecution},
|
||||
execution::{AuthenticatedClient, ExecutionRequestMetadata, StartedExecution},
|
||||
};
|
||||
|
||||
/// 同一请求图内不可逆外部副作用的单调水位
|
||||
@@ -59,37 +59,14 @@ pub struct NestedModelExecutionRequest {
|
||||
pub parent_account: Option<ProviderAccountId>,
|
||||
}
|
||||
|
||||
/// 插件为当前调用或连接显式绑定的模型执行身份
|
||||
///
|
||||
/// Key ID 由插件选择,发起实例由宿主确定
|
||||
/// Core 在绑定时冻结当前
|
||||
/// Key 策略;每个实际模型请求仍独立经过准入、预算、账本与计费
|
||||
pub struct BoundModelExecutionBinding {
|
||||
pub settings: Option<crate::routing::request_settings::RequestSettings>,
|
||||
pub client_key_id: ClientApiKeyId,
|
||||
pub initiating_plugin_instance_id: String,
|
||||
/// 身份随父调用或连接回收;每个模型请求使用独立的执行期限
|
||||
pub cancellation: CancellationToken,
|
||||
/// 宿主保留的扩展调用链,异步观察不能通过新建模型请求清空防递归事实
|
||||
pub extension_scope: super::extensions::ExtensionCallScope,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BoundModelExecutionBinding {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BoundModelExecutionBinding")
|
||||
.field("client_key_id", &self.client_key_id)
|
||||
.field(
|
||||
"initiating_plugin_instance_id",
|
||||
&self.initiating_plugin_instance_id,
|
||||
)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// 显式身份作用域内的一次模型请求
|
||||
///
|
||||
/// 身份与权限可先用于查询;执行链只在真正启动模型请求时建立
|
||||
pub struct BoundModelExecutionRequest {
|
||||
pub context: BoundModelExecutionContext,
|
||||
pub client: AuthenticatedClient,
|
||||
pub initiating_plugin_instance_id: String,
|
||||
pub cancellation: CancellationToken,
|
||||
pub extension_scope: super::extensions::ExtensionCallScope,
|
||||
pub public_model: PublicModelId,
|
||||
pub operation: Operation,
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
@@ -101,7 +78,11 @@ impl fmt::Debug for BoundModelExecutionRequest {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BoundModelExecutionRequest")
|
||||
.field("context", &self.context)
|
||||
.field("client", &self.client)
|
||||
.field(
|
||||
"initiating_plugin_instance_id",
|
||||
&self.initiating_plugin_instance_id,
|
||||
)
|
||||
.field("public_model", &self.public_model)
|
||||
.field("operation", &self.operation.kind())
|
||||
.field("provider", &self.provider)
|
||||
@@ -132,17 +113,12 @@ impl fmt::Debug for NestedModelExecutionRequest {
|
||||
pub trait NestedModelExecutionPort: Send + Sync {
|
||||
fn models(
|
||||
&self,
|
||||
context: BoundModelExecutionContext,
|
||||
client: AuthenticatedClient,
|
||||
cancellation: CancellationToken,
|
||||
protocol: String,
|
||||
client_version: String,
|
||||
) -> BoxFuture<'_, Result<Vec<PublicModelId>, GatewayError>>;
|
||||
|
||||
/// 为插件调用冻结显式执行身份;取消父调用会取消其所有模型执行
|
||||
fn bind(
|
||||
&self,
|
||||
binding: BoundModelExecutionBinding,
|
||||
) -> BoxFuture<'_, Result<BoundModelExecutionContext, GatewayError>>;
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: NestedModelExecutionRequest,
|
||||
|
||||
@@ -34,6 +34,7 @@ use engine::ExecutionStore;
|
||||
use engine::admission::{
|
||||
ClientAdmissionPort, ClientAdmissionRecoveryPort, restore_client_admission_startup,
|
||||
};
|
||||
use engine::authentication::ClientIdentityPort;
|
||||
use engine::continuation::NativeContinuationPort;
|
||||
use engine::execution::{
|
||||
ClientApiKeyUsageSink, ClientKeyVerifier, DefaultExecutionService, ExecutionService,
|
||||
@@ -127,6 +128,7 @@ pub struct CoreBundle {
|
||||
client_key_verifier: Arc<dyn ClientKeyVerifier>,
|
||||
snapshot_control: Arc<dyn SnapshotControl>,
|
||||
account_probe: Arc<dyn AccountProbe>,
|
||||
client_identity: Arc<dyn ClientIdentityPort>,
|
||||
nested_model_execution: Arc<dyn NestedModelExecutionPort>,
|
||||
affinity_lookup: Arc<dyn AffinityLookupPort>,
|
||||
health_probes: Vec<Arc<dyn HealthProbe>>,
|
||||
@@ -181,6 +183,7 @@ impl CoreStartup {
|
||||
let execution: Arc<dyn ExecutionService> = service.clone();
|
||||
let client_key_verifier: Arc<dyn ClientKeyVerifier> = service.clone();
|
||||
let account_probe: Arc<dyn AccountProbe> = service.clone();
|
||||
let client_identity: Arc<dyn ClientIdentityPort> = service.clone();
|
||||
let nested_model_execution: Arc<dyn NestedModelExecutionPort> = service.clone();
|
||||
let affinity_lookup: Arc<dyn AffinityLookupPort> = service;
|
||||
let snapshot_control: Arc<dyn SnapshotControl> = publisher;
|
||||
@@ -191,6 +194,7 @@ impl CoreStartup {
|
||||
client_key_verifier,
|
||||
snapshot_control,
|
||||
account_probe,
|
||||
client_identity,
|
||||
nested_model_execution,
|
||||
affinity_lookup,
|
||||
health_probes,
|
||||
@@ -266,6 +270,11 @@ impl CoreBundle {
|
||||
Arc::clone(&self.account_probe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_identity_port(&self) -> Arc<dyn ClientIdentityPort> {
|
||||
Arc::clone(&self.client_identity)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn nested_model_execution_port(&self) -> Arc<dyn NestedModelExecutionPort> {
|
||||
Arc::clone(&self.nested_model_execution)
|
||||
@@ -395,6 +404,7 @@ pub struct CoreCommandPlaneStartup {
|
||||
pub struct CoreCommandPlaneBundle {
|
||||
control: CoreControlPlaneBundle,
|
||||
account_probe: Arc<dyn AccountProbe>,
|
||||
client_identity: Arc<dyn ClientIdentityPort>,
|
||||
nested_model_execution: Arc<dyn NestedModelExecutionPort>,
|
||||
affinity_lookup: Arc<dyn AffinityLookupPort>,
|
||||
}
|
||||
@@ -425,11 +435,13 @@ impl CoreCommandPlaneStartup {
|
||||
.with_snapshot_refresh(Arc::clone(&control.publisher)),
|
||||
);
|
||||
let account_probe: Arc<dyn AccountProbe> = service.clone();
|
||||
let client_identity: Arc<dyn ClientIdentityPort> = service.clone();
|
||||
let nested_model_execution: Arc<dyn NestedModelExecutionPort> = service.clone();
|
||||
let affinity_lookup: Arc<dyn AffinityLookupPort> = service;
|
||||
Ok(CoreCommandPlaneBundle {
|
||||
control,
|
||||
account_probe,
|
||||
client_identity,
|
||||
nested_model_execution,
|
||||
affinity_lookup,
|
||||
})
|
||||
@@ -452,6 +464,11 @@ impl CoreCommandPlaneBundle {
|
||||
self.control.snapshot_control()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_identity_port(&self) -> Arc<dyn ClientIdentityPort> {
|
||||
Arc::clone(&self.client_identity)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn nested_model_execution_port(&self) -> Arc<dyn NestedModelExecutionPort> {
|
||||
Arc::clone(&self.nested_model_execution)
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
//! 身份、冻结权限与查询不建立模型执行图;实际执行才检查调用链并占用执行资源
|
||||
|
||||
use super::*;
|
||||
use gateway_core::engine::{
|
||||
extensions::{ExtensionCallScope, InvocationOrigin},
|
||||
nested::{BoundModelExecutionRequest, NestedModelExecutionPort},
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
|
||||
#[test]
|
||||
fn identity_queries_keep_frozen_permissions_without_using_execution_resources() {
|
||||
block_on(async {
|
||||
let snapshots = RuntimeSnapshotHandle::new(start_snapshot());
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
snapshots.clone(),
|
||||
store.clone(),
|
||||
ProviderRegistry::new([
|
||||
Arc::new(NativeCatalogProvider { fail: false }) as Arc<dyn Provider>
|
||||
])
|
||||
.unwrap(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
usage.clone(),
|
||||
Arc::new(crate::RecordingDiagnostics::default()),
|
||||
);
|
||||
let settings = service.request_settings().unwrap();
|
||||
let key = ClientApiKeyId::new("key_start_test").unwrap();
|
||||
let client =
|
||||
ClientIdentityPort::resolve_client(&service, key.clone(), Some(settings.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
snapshots.publish(start_snapshot_with_policy(
|
||||
2,
|
||||
false,
|
||||
RateLimits::unlimited(),
|
||||
FastMode::Disabled,
|
||||
));
|
||||
assert!(
|
||||
client
|
||||
.policy()
|
||||
.account_scope()
|
||||
.allows(&ProviderAccountId::new("acct_start").unwrap(),)
|
||||
);
|
||||
assert!(
|
||||
!client
|
||||
.policy()
|
||||
.account_scope()
|
||||
.allows(&ProviderAccountId::new("acct_missing").unwrap(),)
|
||||
);
|
||||
let models = NestedModelExecutionPort::models(
|
||||
&service,
|
||||
client.clone(),
|
||||
CancellationToken::default(),
|
||||
"codex".into(),
|
||||
"0.154.0".into(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(models, vec![PublicModelId::new("gpt-start").unwrap()]);
|
||||
let frozen = ClientIdentityPort::resolve_client(&service, key.clone(), Some(settings))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(frozen.snapshot().revision().get(), 1);
|
||||
let current = ClientIdentityPort::resolve_client(&service, key, None).await;
|
||||
assert!(matches!(current, Err(error) if error.kind() == GatewayErrorKind::Unauthorized));
|
||||
|
||||
let cancellation = CancellationToken::default();
|
||||
cancellation.cancel();
|
||||
let cancelled = NestedModelExecutionPort::models(
|
||||
&service,
|
||||
client,
|
||||
cancellation,
|
||||
"codex".into(),
|
||||
"0.154.0".into(),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(cancelled, Err(error) if error.kind() == GatewayErrorKind::Cancelled));
|
||||
assert!(usage.recorded().is_empty());
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_scope_limits_apply_only_when_starting_a_model() {
|
||||
block_on(async {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
store.clone(),
|
||||
ProviderRegistry::new([
|
||||
Arc::new(NativeCatalogProvider { fail: false }) as Arc<dyn Provider>
|
||||
])
|
||||
.unwrap(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
usage.clone(),
|
||||
Arc::new(crate::RecordingDiagnostics::default()),
|
||||
);
|
||||
let repeated = ExtensionCallScope::default()
|
||||
.extending("entry".into())
|
||||
.unwrap();
|
||||
let exhausted = (0..ExtensionCallScope::MAXIMUM_DEPTH)
|
||||
.fold(ExtensionCallScope::default(), |scope, index| {
|
||||
scope.extending(format!("ancestor-{index}")).unwrap()
|
||||
});
|
||||
for extension_scope in [repeated, exhausted] {
|
||||
let client = ClientIdentityPort::resolve_client(
|
||||
&service,
|
||||
ClientApiKeyId::new("key_start_test").unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = NestedModelExecutionPort::start_bound(
|
||||
&service,
|
||||
BoundModelExecutionRequest {
|
||||
client: client.clone(),
|
||||
initiating_plugin_instance_id: "entry".into(),
|
||||
cancellation: CancellationToken::default(),
|
||||
extension_scope,
|
||||
public_model: PublicModelId::new("gpt-start").unwrap(),
|
||||
operation: start_operation(),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".into(),
|
||||
endpoint: "host.model".into(),
|
||||
transport: ClientTransport::InternalPlugin,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
provider: None,
|
||||
account: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(error) if error.kind() == GatewayErrorKind::PolicyDenied));
|
||||
let models = NestedModelExecutionPort::models(
|
||||
&service,
|
||||
client,
|
||||
CancellationToken::default(),
|
||||
"codex".into(),
|
||||
"0.154.0".into(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(models, vec![PublicModelId::new("gpt-start").unwrap()]);
|
||||
}
|
||||
assert!(usage.recorded().is_empty());
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
});
|
||||
}
|
||||
|
||||
struct ParentAwareAuthentication {
|
||||
calls: AtomicUsize,
|
||||
mapped: bool,
|
||||
}
|
||||
|
||||
impl FrontendAuthenticationPlan for ParentAwareAuthentication {
|
||||
fn authenticate<'a>(
|
||||
&'a self,
|
||||
request: &'a ClientAuthenticationRequest,
|
||||
) -> BoxFuture<'a, Result<FrontendAuthenticationDecision, FrontendAuthenticationError>> {
|
||||
Box::pin(async move {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let origin = request.invocation().expect("parent invocation");
|
||||
assert_eq!(origin.request_id, "http-request");
|
||||
assert_eq!(origin.call_id, "http-call");
|
||||
assert!(origin.extensions.contains("authentication-plugin"));
|
||||
assert!(origin.effects.is_some());
|
||||
assert!(!origin.cancellation.is_cancelled());
|
||||
assert_eq!(request.request_context().unwrap().uri, "/plugin/proxy");
|
||||
assert!(request.settings().is_some());
|
||||
Ok(FrontendAuthenticationDecision::Authenticated {
|
||||
principal: "external-principal".into(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn client_key_id(&self, principal: &str) -> Option<ClientApiKeyId> {
|
||||
assert_eq!(principal, "external-principal");
|
||||
self.mapped
|
||||
.then(|| ClientApiKeyId::new("key_start_test").unwrap())
|
||||
}
|
||||
|
||||
fn exclusive(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_keeps_parent_origin_and_requires_the_frontend_principal_mapping() {
|
||||
block_on(async {
|
||||
for mapped in [true, false] {
|
||||
let authentication = FrontendAuthenticationExtensionIndex::default();
|
||||
let plan = Arc::new(ParentAwareAuthentication {
|
||||
calls: AtomicUsize::new(0),
|
||||
mapped,
|
||||
});
|
||||
let snapshot = start_snapshot().with_extensions(Some(authentication_generation(
|
||||
&authentication,
|
||||
"identity-verification",
|
||||
plan.clone(),
|
||||
)));
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot),
|
||||
store.clone(),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
usage.clone(),
|
||||
Arc::new(crate::RecordingDiagnostics::default()),
|
||||
)
|
||||
.with_frontend_authentication(authentication);
|
||||
let settings = service.request_settings().unwrap();
|
||||
let origin = InvocationOrigin {
|
||||
request_id: "http-request".into(),
|
||||
call_id: "http-call".into(),
|
||||
cancellation: CancellationToken::default(),
|
||||
extensions: plugin_origin_scope()
|
||||
.extending("authentication-plugin".into())
|
||||
.unwrap(),
|
||||
plan: None,
|
||||
effects: Some(Arc::new(
|
||||
gateway_core::engine::nested::ExecutionEffects::default(),
|
||||
)),
|
||||
};
|
||||
let mut cancelled_origin = origin.clone();
|
||||
cancelled_origin.cancellation = CancellationToken::default();
|
||||
cancelled_origin.cancellation.cancel();
|
||||
let cancelled = ClientIdentityPort::verify_client(
|
||||
&service,
|
||||
ClientAuthenticationRequest::new("External fixture")
|
||||
.unwrap()
|
||||
.with_invocation(cancelled_origin)
|
||||
.with_settings(settings.clone()),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(cancelled, Err(error) if error.kind() == GatewayErrorKind::Cancelled));
|
||||
assert_eq!(plan.calls.load(Ordering::SeqCst), 0);
|
||||
let result = ClientIdentityPort::verify_client(
|
||||
&service,
|
||||
ClientAuthenticationRequest::new("External fixture")
|
||||
.unwrap()
|
||||
.with_invocation(origin)
|
||||
.with_settings(settings.clone()),
|
||||
)
|
||||
.await;
|
||||
if mapped {
|
||||
let client = result.unwrap();
|
||||
assert_eq!(client.policy().key_id().as_str(), "key_start_test");
|
||||
assert_eq!(client.request_settings(), Some(&settings));
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(error) if error.kind() == GatewayErrorKind::Unauthorized)
|
||||
);
|
||||
}
|
||||
assert_eq!(plan.calls.load(Ordering::SeqCst), 1);
|
||||
assert!(usage.recorded().is_empty());
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 执行服务的准入、预算、请求记录与终结行为测试
|
||||
|
||||
mod identity;
|
||||
mod outbound;
|
||||
use gateway_core::routing::extensions::{ExtensionSetId, ExtensionSetLease, ExtensionSetReference};
|
||||
|
||||
@@ -12,6 +13,7 @@ use std::{
|
||||
time::{Duration, Instant, SystemTime},
|
||||
};
|
||||
|
||||
use gateway_core::engine::authentication::ClientIdentityPort;
|
||||
use gateway_core::engine::budget::{ClientBudgetCharge, ClientBudgetError, ClientBudgetPort};
|
||||
use gateway_core::error::GatewayError;
|
||||
|
||||
@@ -1215,9 +1217,7 @@ fn plugin_origin_scope() -> gateway_core::engine::extensions::ExtensionCallScope
|
||||
|
||||
#[test]
|
||||
fn explicit_key_model_execution_preserves_http_origin_and_leaves_background_origin_unknown() {
|
||||
use gateway_core::engine::nested::{
|
||||
BoundModelExecutionBinding, BoundModelExecutionRequest, NestedModelExecutionPort,
|
||||
};
|
||||
use gateway_core::engine::nested::{BoundModelExecutionRequest, NestedModelExecutionPort};
|
||||
block_on(async {
|
||||
for has_origin in [true, false] {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
@@ -1234,11 +1234,17 @@ fn explicit_key_model_execution_preserves_http_origin_and_leaves_background_orig
|
||||
Arc::new(crate::RecordingDiagnostics::default()),
|
||||
);
|
||||
let input = request(&service, ClientTransport::InternalPlugin);
|
||||
let context = NestedModelExecutionPort::bind(
|
||||
let client = ClientIdentityPort::resolve_client(
|
||||
&service,
|
||||
BoundModelExecutionBinding {
|
||||
settings: None,
|
||||
client_key_id: ClientApiKeyId::new("key_start_test").unwrap(),
|
||||
ClientApiKeyId::new("key_start_test").unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut started = NestedModelExecutionPort::start_bound(
|
||||
&service,
|
||||
BoundModelExecutionRequest {
|
||||
client,
|
||||
initiating_plugin_instance_id: "origin-fixture".into(),
|
||||
cancellation: Default::default(),
|
||||
extension_scope: if has_origin {
|
||||
@@ -1246,14 +1252,6 @@ fn explicit_key_model_execution_preserves_http_origin_and_leaves_background_orig
|
||||
} else {
|
||||
Default::default()
|
||||
},
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut started = NestedModelExecutionPort::start_bound(
|
||||
&service,
|
||||
BoundModelExecutionRequest {
|
||||
context: context.clone(),
|
||||
public_model: input.public_model,
|
||||
operation: input.operation,
|
||||
metadata: input.metadata,
|
||||
|
||||
@@ -29,7 +29,7 @@ percent-encoding = "2.3.2"
|
||||
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "rustls-tls-native-roots", "socks"] }
|
||||
rustls = { version = "=0.23.46", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
|
||||
rustls-native-certs = "=0.8.3"
|
||||
scraper = "0.27.0"
|
||||
scraper = "0.29.0"
|
||||
semver = "1.0.27"
|
||||
secrecy = "0.10.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
@@ -24,7 +24,7 @@ gateway-plugin-sdk = { path = "../sdk", features = ["io"] }
|
||||
gateway-protocol = { path = "../../gateway-protocol" }
|
||||
hex = "0.4.3"
|
||||
image = { version = "0.25.8", default-features = false, features = ["bmp", "gif", "ico", "jpeg", "png", "webp"] }
|
||||
jsonschema = { version = "0.58.4", default-features = false }
|
||||
jsonschema = { version = "0.60.0", default-features = false }
|
||||
roxmltree = "0.21.1"
|
||||
semver = "1.0.27"
|
||||
secrecy = "0.10.3"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"manifest_schema_versions": [2],
|
||||
"protocol_versions": [5],
|
||||
"protocol_versions": [7],
|
||||
"capabilities": [
|
||||
{ "capability": "frontend_authentication", "versions": [2] },
|
||||
{ "capability": "scheduler", "versions": [3] },
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
//! 编译插件模型目录贡献并校验能力绑定
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
plugins::instances::{PluginCapabilityBinding, PluginInstance},
|
||||
@@ -23,7 +21,7 @@ pub(crate) async fn prepare(
|
||||
let reply = session
|
||||
.call(
|
||||
"model_catalog.register",
|
||||
session.context(Stage::Registration, Duration::from_secs(5)),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
Vec::new(),
|
||||
)
|
||||
|
||||
@@ -48,12 +48,13 @@ impl PluginCommandSession {
|
||||
let payload = serde_json::to_vec(&invocation)
|
||||
.map_err(|_| AdminError::invalid("插件命令参数无法编码"))?;
|
||||
let mut saved_accounts = 0;
|
||||
let deadline = tokio::time::Instant::now() + command.session.budget(Stage::CommandLine);
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
() = cancellation.cancelled() => Ok(Err("调用已取消")),
|
||||
result = tokio::time::timeout(
|
||||
self.limits.maximum_call_timeout,
|
||||
command.execute(payload, self.limits, &mut saved_accounts),
|
||||
result = tokio::time::timeout_at(
|
||||
deadline,
|
||||
command.execute(payload, self.limits, deadline, &mut saved_accounts),
|
||||
) => result,
|
||||
};
|
||||
match result {
|
||||
@@ -76,11 +77,14 @@ impl PluginCommand {
|
||||
&self,
|
||||
payload: Vec<u8>,
|
||||
limits: RpcLimits,
|
||||
deadline: tokio::time::Instant,
|
||||
saved_accounts: &mut usize,
|
||||
) -> Result<PluginCommandOutput, &'static str> {
|
||||
let mut context = self
|
||||
.session
|
||||
.context(Stage::CommandLine, limits.maximum_call_timeout);
|
||||
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err("超过命令总期限");
|
||||
}
|
||||
let mut context = self.session.context(Stage::CommandLine, remaining);
|
||||
context.request_id = Some(uuid::Uuid::new_v4().to_string());
|
||||
let scope = self
|
||||
.callbacks
|
||||
@@ -135,6 +139,7 @@ fn rpc_failure(error: RpcError) -> &'static str {
|
||||
}
|
||||
RpcError::Capacity | RpcError::Overloaded => "插件调用容量不足",
|
||||
RpcError::Context(_) => "调用上下文无效",
|
||||
RpcError::Budget(_) => "调用预算冲突",
|
||||
RpcError::Remote(_) => "插件拒绝命令",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use super::PluginCommand;
|
||||
use crate::{RpcSession, callback::PluginCallbacks};
|
||||
use gateway_admin::model::{AdminError, plugins::instances::PluginInstance};
|
||||
use gateway_plugin_sdk::{Capability, Manifest, Stage, call::management::CommandRegistration};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::sync::Arc;
|
||||
|
||||
const MAXIMUM_REGISTRATION_BYTES: usize = 64 * 1024;
|
||||
|
||||
@@ -23,7 +23,7 @@ pub(crate) async fn prepare(
|
||||
let reply = session
|
||||
.call(
|
||||
"command_line.register",
|
||||
session.context(Stage::Registration, Duration::from_secs(5)),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
Vec::new(),
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 编译插件客户端认证绑定,并将认证结果映射为宿主身份
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Duration};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::model::{
|
||||
@@ -26,8 +26,6 @@ use gateway_plugin_sdk::{
|
||||
|
||||
use crate::{RpcSession, callback::PluginCallbacks};
|
||||
|
||||
const IDENTIFIER_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct FrontendAuthenticationEntry {
|
||||
identities: BTreeMap<String, ClientApiKeyId>,
|
||||
@@ -47,7 +45,7 @@ pub(crate) async fn prepare_entry(
|
||||
let reply = session
|
||||
.call(
|
||||
"frontend_auth.identifier",
|
||||
session.context(Stage::Registration, IDENTIFIER_TIMEOUT),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
Vec::new(),
|
||||
)
|
||||
@@ -167,19 +165,16 @@ fn valid_identifier(value: &str) -> bool {
|
||||
|
||||
pub(crate) struct PluginFrontendAuthenticationPlan {
|
||||
entry: FrontendAuthenticationEntry,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PluginFrontendAuthenticationPlan {
|
||||
pub(crate) fn compile(
|
||||
mut entries: Vec<FrontendAuthenticationEntry>,
|
||||
maximum_call_timeout: Duration,
|
||||
) -> Result<Option<Arc<Self>>, AdminError> {
|
||||
match entries.len() {
|
||||
0 => Ok(None),
|
||||
1 => Ok(Some(Arc::new(Self {
|
||||
entry: entries.pop().expect("one entry was checked"),
|
||||
timeout: maximum_call_timeout.min(Duration::from_secs(30)),
|
||||
}))),
|
||||
_ => Err(AdminError::invalid("同一发布集合只能启用一个入口认证插件")),
|
||||
}
|
||||
@@ -197,23 +192,32 @@ impl FrontendAuthenticationPlan for PluginFrontendAuthenticationPlan {
|
||||
.invocation
|
||||
.as_ref()
|
||||
.ok_or(FrontendAuthenticationError)?;
|
||||
let mut context = session.context(Stage::Authentication, self.timeout);
|
||||
let mut context =
|
||||
session.context(Stage::Authentication, session.budget(Stage::Authentication));
|
||||
context.request = crate::adapter::request_context(request.request_context());
|
||||
context.request_id = request.invocation().map(|origin| origin.request_id.clone());
|
||||
let _scope = callbacks
|
||||
.prepare_frontend_authentication(&context, request.shared_request_context())
|
||||
.prepare_frontend_authentication(&context, request)
|
||||
.map_err(|_| FrontendAuthenticationError)?;
|
||||
let input = FrontendAuthenticationRequest {
|
||||
authorization: request.authorization().to_owned(),
|
||||
};
|
||||
let reply = session
|
||||
.call(
|
||||
"frontend_auth.authenticate",
|
||||
context,
|
||||
serde_json::json!({}),
|
||||
serde_json::to_vec(&input).map_err(|_| FrontendAuthenticationError)?,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| FrontendAuthenticationError)?;
|
||||
let invocation = session.call(
|
||||
"frontend_auth.authenticate",
|
||||
context,
|
||||
serde_json::json!({}),
|
||||
serde_json::to_vec(&input).map_err(|_| FrontendAuthenticationError)?,
|
||||
);
|
||||
let reply = if let Some(origin) = request.invocation() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = origin.cancellation.cancelled() => return Err(FrontendAuthenticationError),
|
||||
reply = invocation => reply,
|
||||
}
|
||||
} else {
|
||||
invocation.await
|
||||
}
|
||||
.map_err(|_| FrontendAuthenticationError)?;
|
||||
let result: FrontendAuthenticationResult =
|
||||
serde_json::from_slice(&reply.payload).map_err(|_| FrontendAuthenticationError)?;
|
||||
match result {
|
||||
|
||||
@@ -165,9 +165,10 @@ impl ManagementEntry {
|
||||
{
|
||||
return Err(invalid_state());
|
||||
}
|
||||
let mut context = self
|
||||
.session
|
||||
.context(Stage::PublicManagement, limits.maximum_call_timeout);
|
||||
let mut context = self.session.context(
|
||||
Stage::PublicManagement,
|
||||
self.session.budget(Stage::PublicManagement),
|
||||
);
|
||||
context.request = crate::adapter::request_context(request.request_context.as_deref());
|
||||
context.request_id = Some(request.request_id);
|
||||
let _scope = self
|
||||
|
||||
@@ -98,7 +98,7 @@ impl ManagementEntry {
|
||||
.map_err(|_| AdminError::invalid("插件管理请求无法编码"))?;
|
||||
let mut context = self
|
||||
.session
|
||||
.context(Stage::Management, limits.maximum_call_timeout);
|
||||
.context(Stage::Management, self.session.budget(Stage::Management));
|
||||
context.request = crate::adapter::request_context(request.request_context.as_deref());
|
||||
context.request_id = Some(request.request_id);
|
||||
let _scope = self
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 将插件管理声明编译为页面、路由与资源入口
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Duration};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
@@ -34,7 +34,7 @@ pub(crate) async fn prepare(
|
||||
let reply = session
|
||||
.call(
|
||||
"management.register",
|
||||
session.context(Stage::Registration, Duration::from_secs(5)),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
Vec::new(),
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ use gateway_core::engine::middleware::{
|
||||
MiddlewareRequest, MiddlewareResponse,
|
||||
};
|
||||
use gateway_core::middleware::{Middleware, Next};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::sync::Arc;
|
||||
|
||||
// 协议适配只提供选择条件与单层调用;顺序、上下文捕获和续体拼接共用这一个入口
|
||||
type Invoke<Context, Input, Output, Error> = fn(
|
||||
@@ -15,7 +15,6 @@ type Invoke<Context, Input, Output, Error> = fn(
|
||||
Context,
|
||||
Input,
|
||||
Next<Input, Output, Error>,
|
||||
Duration,
|
||||
) -> BoxFuture<'static, Result<Output, Error>>;
|
||||
|
||||
impl PluginMiddlewarePlan {
|
||||
@@ -42,8 +41,7 @@ impl PluginMiddlewarePlan {
|
||||
.map(|entry| -> Middleware<Input, Output, Error> {
|
||||
let entry = Arc::clone(entry);
|
||||
let context = context.clone();
|
||||
let timeout = self.middleware_timeout;
|
||||
Box::new(move |input, next| invoke(entry, context, input, next, timeout))
|
||||
Box::new(move |input, next| invoke(entry, context, input, next))
|
||||
})
|
||||
.collect();
|
||||
gateway_core::middleware::compose(layers, move |input| next.run(input))
|
||||
@@ -70,10 +68,8 @@ impl MiddlewarePlan for PluginMiddlewarePlan {
|
||||
MiddlewareMount::Outbound,
|
||||
context,
|
||||
outbound_matches,
|
||||
|entry, context, request, next, timeout| {
|
||||
Box::pin(super::outbound::invoke(
|
||||
entry, context, request, next, timeout,
|
||||
))
|
||||
|entry, context, request, next| {
|
||||
Box::pin(super::outbound::invoke(entry, context, request, next))
|
||||
},
|
||||
next,
|
||||
)
|
||||
@@ -99,8 +95,8 @@ impl MiddlewarePlan for PluginMiddlewarePlan {
|
||||
MiddlewareMount::Service,
|
||||
context,
|
||||
|entry, context| !context.extensions.contains(&entry.instance_id),
|
||||
|entry, context, input, next, timeout| {
|
||||
Box::pin(super::service::invoke(entry, context, input, next, timeout))
|
||||
|entry, context, input, next| {
|
||||
Box::pin(super::service::invoke(entry, context, input, next))
|
||||
},
|
||||
next,
|
||||
)
|
||||
@@ -123,10 +119,8 @@ impl MiddlewarePlan for PluginMiddlewarePlan {
|
||||
MiddlewareMount::WebSocket,
|
||||
context,
|
||||
|_entry, _context| true,
|
||||
|entry, context, message, next, timeout| {
|
||||
Box::pin(super::websocket::invoke(
|
||||
entry, context, message, next, timeout,
|
||||
))
|
||||
|entry, context, message, next| {
|
||||
Box::pin(super::websocket::invoke(entry, context, message, next))
|
||||
},
|
||||
next,
|
||||
)
|
||||
@@ -147,8 +141,8 @@ impl MiddlewarePlan for PluginMiddlewarePlan {
|
||||
MiddlewareMount::Http,
|
||||
context,
|
||||
|entry, context| !context.extensions.contains(&entry.instance_id),
|
||||
|entry, context, request, next, timeout| {
|
||||
Box::pin(super::http::invoke(entry, context, request, next, timeout))
|
||||
|entry, context, request, next| {
|
||||
Box::pin(super::http::invoke(entry, context, request, next))
|
||||
},
|
||||
next,
|
||||
)
|
||||
@@ -173,10 +167,8 @@ impl MiddlewarePlan for PluginMiddlewarePlan {
|
||||
context.model(),
|
||||
)
|
||||
},
|
||||
|entry, context, request, next, timeout| {
|
||||
Box::pin(super::request::invoke(
|
||||
entry, context, request, next, timeout,
|
||||
))
|
||||
|entry, context, request, next| {
|
||||
Box::pin(super::request::invoke(entry, context, request, next))
|
||||
},
|
||||
next,
|
||||
)
|
||||
|
||||
@@ -20,7 +20,6 @@ pub(super) async fn invoke(
|
||||
context: core::Context,
|
||||
request: core::Request,
|
||||
next: core::Next,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<core::Response, MiddlewareError> {
|
||||
let Some(ports) = &entry.invocation else {
|
||||
return if entry.failure_policy == PluginFailurePolicy::Delegate {
|
||||
@@ -35,7 +34,9 @@ pub(super) async fn invoke(
|
||||
.and_then(|settings| settings.runtime.as_ref())
|
||||
.map_or(serde_json::Value::Null, |settings| settings.inspect());
|
||||
let (invocation, request) = Invocation::new(request, next, entry.instance_id.clone())?;
|
||||
let mut call = ports.session.context(Stage::Http, timeout);
|
||||
let mut call = ports
|
||||
.session
|
||||
.context(Stage::Http, ports.session.budget(Stage::Http));
|
||||
call.request = crate::adapter::request_context(context.extensions.request());
|
||||
call.request_id = Some(context.request_id.clone());
|
||||
call.resource_stream = true;
|
||||
@@ -50,7 +51,9 @@ pub(super) async fn invoke(
|
||||
request_id: context.request_id.clone(),
|
||||
call_id: context.call_id.clone(),
|
||||
cancellation: context.cancellation.clone(),
|
||||
plan,
|
||||
plan: Some(plan),
|
||||
extensions: context.extensions.clone(),
|
||||
effects: None,
|
||||
}),
|
||||
context.extensions.clone(),
|
||||
invocation.clone(),
|
||||
|
||||
@@ -18,7 +18,6 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -168,7 +167,6 @@ fn compile_entry(
|
||||
|
||||
pub(crate) struct PluginMiddlewarePlan {
|
||||
middleware: BTreeMap<MiddlewareMount, Box<[Arc<MiddlewareEntry>]>>,
|
||||
middleware_timeout: Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PluginMiddlewarePlan {
|
||||
@@ -181,10 +179,7 @@ impl fmt::Debug for PluginMiddlewarePlan {
|
||||
}
|
||||
|
||||
impl PluginMiddlewarePlan {
|
||||
pub(crate) fn compile(
|
||||
mut entries: Vec<MiddlewareEntry>,
|
||||
maximum_call_timeout: Duration,
|
||||
) -> Option<Arc<Self>> {
|
||||
pub(crate) fn compile(mut entries: Vec<MiddlewareEntry>) -> Option<Arc<Self>> {
|
||||
if entries.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -208,7 +203,6 @@ impl PluginMiddlewarePlan {
|
||||
.into_iter()
|
||||
.map(|(mount, entries)| (mount, entries.into_boxed_slice()))
|
||||
.collect(),
|
||||
middleware_timeout: maximum_call_timeout,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 在 Provider 发送前组合完整出站请求,并沿用模型数据面的调用资源
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_admin::model::plugins::instances::PluginFailurePolicy;
|
||||
use gateway_core::engine::middleware::{MiddlewareContext, MiddlewareError, outbound as core};
|
||||
@@ -17,7 +17,6 @@ pub(super) async fn invoke(
|
||||
context: MiddlewareContext,
|
||||
request: core::Request,
|
||||
next: core::Next,
|
||||
timeout: Duration,
|
||||
) -> Result<core::Request, MiddlewareError> {
|
||||
let Some(ports) = &entry.invocation else {
|
||||
return if entry.failure_policy == PluginFailurePolicy::Delegate {
|
||||
@@ -54,7 +53,12 @@ pub(super) async fn invoke(
|
||||
.get(),
|
||||
},
|
||||
};
|
||||
let mut call = ports.session.context(Stage::Outbound, timeout);
|
||||
let mut call = ports.session.context(
|
||||
Stage::Outbound,
|
||||
context
|
||||
.deadline()
|
||||
.bounded(ports.session.budget(Stage::Outbound)),
|
||||
);
|
||||
call.request = crate::adapter::request_context(context.extension_scope().request());
|
||||
call.request_id = Some(input.metadata.request_id.clone());
|
||||
call.attempt_id = Some(format!(
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
//! 请求与尝试中间件的 RPC 调用、失败回退和响应正文生命周期
|
||||
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
@@ -37,7 +34,6 @@ pub(super) async fn invoke(
|
||||
context: MiddlewareContext,
|
||||
request: MiddlewareRequest,
|
||||
next: MiddlewareNext,
|
||||
maximum_timeout: Duration,
|
||||
) -> Result<MiddlewareResponse, MiddlewareError> {
|
||||
let Some(invocation_ports) = &entry.invocation else {
|
||||
return if entry.failure_policy
|
||||
@@ -48,10 +44,7 @@ pub(super) async fn invoke(
|
||||
Err(MiddlewareError::Fault)
|
||||
};
|
||||
};
|
||||
if maximum_timeout.is_zero()
|
||||
|| context.deadline().is_elapsed()
|
||||
|| context.cancellation().is_cancelled()
|
||||
{
|
||||
if context.deadline().is_elapsed() || context.cancellation().is_cancelled() {
|
||||
return Err(MiddlewareError::Fault);
|
||||
}
|
||||
let invocation = MiddlewareInvocation::new(&context, request, next, entry.instance_id.clone());
|
||||
@@ -59,18 +52,21 @@ pub(super) async fn invoke(
|
||||
Ok(projection) => projection,
|
||||
Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context(None)).await,
|
||||
};
|
||||
let stage = match entry.mount {
|
||||
MiddlewareMount::Http
|
||||
| MiddlewareMount::WebSocket
|
||||
| MiddlewareMount::Service
|
||||
| MiddlewareMount::Outbound => {
|
||||
return Err(MiddlewareError::InvalidState);
|
||||
}
|
||||
MiddlewareMount::Request => Stage::Request,
|
||||
MiddlewareMount::Attempt => Stage::Attempt,
|
||||
};
|
||||
let mut call_context = invocation_ports.session.context(
|
||||
match entry.mount {
|
||||
MiddlewareMount::Http
|
||||
| MiddlewareMount::WebSocket
|
||||
| MiddlewareMount::Service
|
||||
| MiddlewareMount::Outbound => {
|
||||
return Err(MiddlewareError::InvalidState);
|
||||
}
|
||||
MiddlewareMount::Request => Stage::Request,
|
||||
MiddlewareMount::Attempt => Stage::Attempt,
|
||||
},
|
||||
maximum_timeout,
|
||||
stage,
|
||||
context
|
||||
.deadline()
|
||||
.bounded(invocation_ports.session.budget(stage)),
|
||||
);
|
||||
call_context.resource_stream = true;
|
||||
call_context.request = crate::adapter::request_context(context.extension_scope().request());
|
||||
|
||||
@@ -16,7 +16,6 @@ pub(super) async fn invoke(
|
||||
context: core::Context,
|
||||
input: core::Value,
|
||||
next: core::Next,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<core::Value, core::Error> {
|
||||
let Some(ports) = &entry.invocation else {
|
||||
return if entry.failure_policy == PluginFailurePolicy::Delegate {
|
||||
@@ -26,7 +25,9 @@ pub(super) async fn invoke(
|
||||
};
|
||||
};
|
||||
let invocation = Invocation::new(next);
|
||||
let mut call = ports.session.context(Stage::Service, timeout);
|
||||
let mut call = ports
|
||||
.session
|
||||
.context(Stage::Service, ports.session.budget(Stage::Service));
|
||||
call.request = crate::adapter::request_context(context.extensions.request());
|
||||
call.request_id = Some(context.request_id.clone());
|
||||
let _binding = ports
|
||||
@@ -37,7 +38,9 @@ pub(super) async fn invoke(
|
||||
request_id: context.request_id.clone(),
|
||||
call_id: context.call_id.clone(),
|
||||
cancellation: context.cancellation.clone(),
|
||||
plan: context.plan.clone(),
|
||||
plan: Some(context.plan.clone()),
|
||||
extensions: context.extensions.clone(),
|
||||
effects: None,
|
||||
}),
|
||||
context.extensions.clone(),
|
||||
invocation,
|
||||
|
||||
@@ -16,7 +16,6 @@ pub(super) async fn invoke(
|
||||
context: core::Context,
|
||||
message: core::Message,
|
||||
next: core::Next,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Option<core::Message>, MiddlewareError> {
|
||||
let Some(ports) = &entry.invocation else {
|
||||
return if entry.failure_policy == PluginFailurePolicy::Delegate {
|
||||
@@ -44,7 +43,9 @@ pub(super) async fn invoke(
|
||||
.encode(message)
|
||||
.map_err(|_| MiddlewareError::Fault)?,
|
||||
};
|
||||
let mut call = ports.session.context(Stage::WebSocket, timeout);
|
||||
let mut call = ports
|
||||
.session
|
||||
.context(Stage::WebSocket, ports.session.budget(Stage::WebSocket));
|
||||
call.request = crate::adapter::request_context(context.extensions.request());
|
||||
call.request_id = Some(context.connection_id.clone());
|
||||
let _binding = ports
|
||||
@@ -55,7 +56,9 @@ pub(super) async fn invoke(
|
||||
request_id: context.connection_id.clone(),
|
||||
call_id: call.resource_scope_id.clone(),
|
||||
cancellation: context.cancellation.clone(),
|
||||
plan,
|
||||
plan: Some(plan),
|
||||
extensions: context.extensions.clone(),
|
||||
effects: None,
|
||||
}),
|
||||
context.extensions,
|
||||
invocation.clone(),
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use gateway_admin::model::{
|
||||
@@ -31,7 +31,6 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
|
||||
|
||||
use crate::{RpcSession, adapter::scope::BindingScope};
|
||||
|
||||
const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const OBSERVATION_ENVELOPE_RESERVE: usize = 16 * 1024;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -57,10 +56,11 @@ impl ObserverEntry {
|
||||
event: Event,
|
||||
payload: Vec<u8>,
|
||||
extensions: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
timeout: Duration,
|
||||
) {
|
||||
let request_id = event.request_id();
|
||||
let mut context = self.session.context(Stage::Observation, timeout);
|
||||
let mut context = self
|
||||
.session
|
||||
.context(Stage::Observation, self.session.budget(Stage::Observation));
|
||||
context.request = crate::adapter::request_context(extensions.request());
|
||||
context.request_id = Some(request_id.to_owned());
|
||||
let Ok(_scope) = self.callbacks.prepare_extensions(&context, extensions) else {
|
||||
@@ -166,7 +166,6 @@ pub(crate) struct PluginObserverPlan {
|
||||
websocket_queues: Arc<Mutex<BTreeMap<String, mpsc::Sender<QueuedWebSocketObservation>>>>,
|
||||
websocket_queue_capacity: usize,
|
||||
maximum_websocket_payload_bytes: usize,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
struct QueuedWebSocketObservation {
|
||||
@@ -180,7 +179,6 @@ impl PluginObserverPlan {
|
||||
pub(crate) fn compile(
|
||||
mut entries: Vec<ObserverEntry>,
|
||||
maximum_calls: usize,
|
||||
maximum_call_timeout: Duration,
|
||||
maximum_buffered_body_bytes: usize,
|
||||
) -> Option<Arc<Self>> {
|
||||
if entries.is_empty() {
|
||||
@@ -205,7 +203,6 @@ impl PluginObserverPlan {
|
||||
websocket_queues: Arc::new(Mutex::new(BTreeMap::new())),
|
||||
websocket_queue_capacity: maximum_calls.clamp(1, 256),
|
||||
maximum_websocket_payload_bytes,
|
||||
timeout: maximum_call_timeout.min(OBSERVATION_TIMEOUT),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -326,7 +323,6 @@ impl RequestObserverPlan for PluginObserverPlan {
|
||||
return;
|
||||
};
|
||||
let entries = Arc::clone(&self.entries);
|
||||
let timeout = self.timeout;
|
||||
handle.spawn(async move {
|
||||
let _generation = generation;
|
||||
let _slot = slot;
|
||||
@@ -342,7 +338,6 @@ impl RequestObserverPlan for PluginObserverPlan {
|
||||
Event::RequestCompleted(Box::new(wire_observation(&observation))),
|
||||
Vec::new(),
|
||||
observation.extension_scope().clone(),
|
||||
timeout,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -372,10 +367,8 @@ impl PluginObserverPlan {
|
||||
let (sender, receiver) = mpsc::channel(self.websocket_queue_capacity);
|
||||
queues.insert(request_id.to_owned(), sender.clone());
|
||||
let entries = Arc::clone(&self.entries);
|
||||
let timeout = self.timeout;
|
||||
handle.spawn(dispatch_websocket_queue(
|
||||
entries,
|
||||
timeout,
|
||||
generation,
|
||||
worker_slot,
|
||||
receiver,
|
||||
@@ -386,7 +379,6 @@ impl PluginObserverPlan {
|
||||
|
||||
async fn dispatch_websocket_queue(
|
||||
entries: Arc<[ObserverEntry]>,
|
||||
timeout: Duration,
|
||||
generation: ExtensionSetReference,
|
||||
worker_slot: OwnedSemaphorePermit,
|
||||
mut receiver: mpsc::Receiver<QueuedWebSocketObservation>,
|
||||
@@ -422,7 +414,6 @@ async fn dispatch_websocket_queue(
|
||||
Event::WebSocketResponse(Box::new(request)),
|
||||
queued.payload.to_vec(),
|
||||
queued.observation.extension_scope().clone(),
|
||||
timeout,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
mod retry;
|
||||
mod route_schedule;
|
||||
|
||||
use std::{fmt, sync::Arc, time::Duration};
|
||||
use std::{fmt, sync::Arc};
|
||||
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
@@ -13,8 +13,6 @@ use gateway_plugin_sdk::{Capability, Manifest, Stage};
|
||||
|
||||
use crate::{RpcSession, adapter::scope::BindingScope, callback::PluginCallbacks};
|
||||
|
||||
const POLICY_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum PolicyEntry {
|
||||
Router(ModelRouterEntry),
|
||||
@@ -219,7 +217,6 @@ pub(crate) struct PluginRequestPolicyPlan {
|
||||
routers: Arc<[ModelRouterEntry]>,
|
||||
schedulers: Arc<[AccountSchedulerEntry]>,
|
||||
retries: Arc<[RetryEntry]>,
|
||||
policy_timeout: Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PluginRequestPolicyPlan {
|
||||
@@ -234,10 +231,7 @@ impl fmt::Debug for PluginRequestPolicyPlan {
|
||||
}
|
||||
|
||||
impl PluginRequestPolicyPlan {
|
||||
pub(crate) fn compile(
|
||||
entries: Vec<PolicyEntry>,
|
||||
maximum_call_timeout: Duration,
|
||||
) -> Result<Option<Arc<Self>>, AdminError> {
|
||||
pub(crate) fn compile(entries: Vec<PolicyEntry>) -> Result<Option<Arc<Self>>, AdminError> {
|
||||
let mut routers = Vec::new();
|
||||
let mut schedulers = Vec::new();
|
||||
let mut retries = Vec::new();
|
||||
@@ -277,7 +271,6 @@ impl PluginRequestPolicyPlan {
|
||||
routers: routers.into(),
|
||||
schedulers: schedulers.into(),
|
||||
retries: retries.into(),
|
||||
policy_timeout: maximum_call_timeout.min(POLICY_TIMEOUT),
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,11 +17,10 @@ use super::RetryEntry;
|
||||
pub(super) fn decide(
|
||||
entries: Arc<[RetryEntry]>,
|
||||
input: RetryInput,
|
||||
timeout: Duration,
|
||||
) -> BoxFuture<'static, Result<RetryDecision, RequestPolicyFault>> {
|
||||
Box::pin(async move {
|
||||
let started = std::time::Instant::now();
|
||||
for entry in entries.iter().filter(|entry| {
|
||||
let started = tokio::time::Instant::now();
|
||||
let matched = entries.iter().filter(|entry| {
|
||||
!input.extension_scope.contains(&entry.instance_id)
|
||||
&& entry.scope.matches_processing(
|
||||
&input.client_key_id,
|
||||
@@ -29,23 +28,32 @@ pub(super) fn decide(
|
||||
Some(&input.facts.provider),
|
||||
input.facts.model.as_deref(),
|
||||
)
|
||||
}) {
|
||||
});
|
||||
let timeout = matched
|
||||
.clone()
|
||||
.filter_map(|entry| entry.invocation.as_ref())
|
||||
.map(|invocation| invocation.session.budget(Stage::Retry))
|
||||
.max()
|
||||
.unwrap_or_default()
|
||||
.min(input.facts.remaining_deadline);
|
||||
for entry in matched {
|
||||
// 整条委托链共享预算,不能每个插件重新取得完整等待时间
|
||||
let remaining = timeout
|
||||
.min(input.facts.remaining_deadline)
|
||||
.saturating_sub(started.elapsed());
|
||||
let remaining = timeout.saturating_sub(started.elapsed());
|
||||
if remaining.is_zero() {
|
||||
break;
|
||||
}
|
||||
match call(entry, &input, remaining).await {
|
||||
Ok(RetryDecision::Delegate) => {}
|
||||
Ok(decision) => return Ok(decision),
|
||||
Err(()) => tracing::warn!(
|
||||
plugin_id = entry.plugin_id,
|
||||
instance_id = entry.instance_id,
|
||||
request_id = input.request_id.as_str(),
|
||||
"插件重试策略无效或调用失败,委托后续策略"
|
||||
),
|
||||
match tokio::time::timeout(remaining, call(entry, &input, remaining)).await {
|
||||
Err(_) => break,
|
||||
Ok(result) => match result {
|
||||
Ok(RetryDecision::Delegate) => {}
|
||||
Ok(decision) => return Ok(decision),
|
||||
Err(()) => tracing::warn!(
|
||||
plugin_id = entry.plugin_id,
|
||||
instance_id = entry.instance_id,
|
||||
request_id = input.request_id.as_str(),
|
||||
"插件重试策略无效或调用失败,委托后续策略"
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
Ok(RetryDecision::Delegate)
|
||||
@@ -79,7 +87,10 @@ async fn call(
|
||||
.unwrap_or(u64::MAX),
|
||||
allowed_actions,
|
||||
};
|
||||
let mut context = invocation.session.context(Stage::Retry, timeout);
|
||||
let mut context = invocation.session.context(
|
||||
Stage::Retry,
|
||||
timeout.min(invocation.session.budget(Stage::Retry)),
|
||||
);
|
||||
context.request = crate::adapter::request_context(input.extension_scope.request());
|
||||
context.request_id = Some(request.request_id.clone());
|
||||
context.attempt_id = Some(format!("{}:{}", request.request_id, request.attempt_index));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 插件模型路由与账号调度调用,以及请求事实的线协议投影
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use futures::future::BoxFuture;
|
||||
@@ -38,7 +38,7 @@ impl RequestPolicyPlan for PluginRequestPolicyPlan {
|
||||
input: gateway_core::engine::policy::RetryInput,
|
||||
) -> BoxFuture<'static, Result<gateway_core::engine::policy::RetryDecision, RequestPolicyFault>>
|
||||
{
|
||||
super::retry::decide(self.retries.clone(), input, self.policy_timeout)
|
||||
super::retry::decide(self.retries.clone(), input)
|
||||
}
|
||||
|
||||
fn route_model(
|
||||
@@ -46,12 +46,11 @@ impl RequestPolicyPlan for PluginRequestPolicyPlan {
|
||||
input: ModelRouteInput,
|
||||
) -> BoxFuture<'static, Result<ModelRouteDecision, RequestPolicyFault>> {
|
||||
let routers = std::sync::Arc::clone(&self.routers);
|
||||
let timeout = self.policy_timeout;
|
||||
Box::pin(async move {
|
||||
for entry in routers.iter().filter(|entry| {
|
||||
!input.suppresses_plugin(&entry.instance_id) && entry.scope.matches_route(&input)
|
||||
}) {
|
||||
let result = route_with_entry(entry, &input, timeout).await;
|
||||
let result = route_with_entry(entry, &input).await;
|
||||
match result {
|
||||
Ok(ModelRouteDecision::Unhandled) => continue,
|
||||
Ok(decision) => return Ok(decision),
|
||||
@@ -78,14 +77,13 @@ impl RequestPolicyPlan for PluginRequestPolicyPlan {
|
||||
input: AccountScheduleInput,
|
||||
) -> BoxFuture<'static, Result<AccountScheduleDecision, RequestPolicyFault>> {
|
||||
let schedulers = std::sync::Arc::clone(&self.schedulers);
|
||||
let timeout = self.policy_timeout;
|
||||
Box::pin(async move {
|
||||
let Some(entry) = schedulers.iter().find(|entry| {
|
||||
!input.suppresses_plugin(&entry.instance_id) && entry.scope.matches_schedule(&input)
|
||||
}) else {
|
||||
return Ok(AccountScheduleDecision::Delegate);
|
||||
};
|
||||
match schedule_with_entry(entry, &input, timeout).await {
|
||||
match schedule_with_entry(entry, &input).await {
|
||||
Ok(decision) => Ok(decision),
|
||||
Err(())
|
||||
if entry.failure_policy
|
||||
@@ -108,7 +106,6 @@ impl RequestPolicyPlan for PluginRequestPolicyPlan {
|
||||
async fn route_with_entry(
|
||||
entry: &ModelRouterEntry,
|
||||
input: &ModelRouteInput,
|
||||
timeout: Duration,
|
||||
) -> Result<ModelRouteDecision, ()> {
|
||||
let invocation = entry.invocation.as_ref().ok_or(())?;
|
||||
let projection = operation_projection(input.operation())?;
|
||||
@@ -126,7 +123,9 @@ async fn route_with_entry(
|
||||
.collect(),
|
||||
headers,
|
||||
};
|
||||
let mut context = invocation.session.context(Stage::Routing, timeout);
|
||||
let mut context = invocation
|
||||
.session
|
||||
.context(Stage::Routing, invocation.session.budget(Stage::Routing));
|
||||
context.request = crate::adapter::request_context(input.extension_scope().request());
|
||||
context.request_id = Some(input.request_id().as_str().to_owned());
|
||||
let _network_scope = invocation
|
||||
@@ -172,7 +171,6 @@ async fn route_with_entry(
|
||||
async fn schedule_with_entry(
|
||||
entry: &AccountSchedulerEntry,
|
||||
input: &AccountScheduleInput,
|
||||
timeout: Duration,
|
||||
) -> Result<AccountScheduleDecision, ()> {
|
||||
let invocation = entry.invocation.as_ref().ok_or(())?;
|
||||
let candidates = input.candidates();
|
||||
@@ -197,7 +195,10 @@ async fn schedule_with_entry(
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let mut context = invocation.session.context(Stage::Scheduling, timeout);
|
||||
let mut context = invocation.session.context(
|
||||
Stage::Scheduling,
|
||||
invocation.session.budget(Stage::Scheduling),
|
||||
);
|
||||
context.request = crate::adapter::request_context(input.extension_scope().request());
|
||||
context.request_id = Some(input.request_id().as_str().to_owned());
|
||||
context.attempt_id = Some(format!(
|
||||
|
||||
@@ -167,7 +167,7 @@ impl Execution {
|
||||
let timeout = invocation
|
||||
.context
|
||||
.deadline()
|
||||
.bounded(self.adapter.session.maximum_call_timeout());
|
||||
.bounded(self.adapter.session.budget(Stage::Upstream));
|
||||
if timeout.is_zero() || invocation.context.cancellation().is_cancelled() {
|
||||
return Err(ProviderError::new(
|
||||
ProviderErrorKind::Cancelled,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 校验并准备插件上游适配声明与能力绑定
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
@@ -56,7 +56,7 @@ pub(crate) async fn prepare(
|
||||
let reply = session
|
||||
.call(
|
||||
REGISTER_METHOD,
|
||||
session.context(Stage::Registration, Duration::from_secs(5)),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
vec![],
|
||||
)
|
||||
|
||||
@@ -20,8 +20,8 @@ use gateway_core::{
|
||||
use gateway_plugin_sdk::{
|
||||
CallContext, ErrorCode, PluginFault,
|
||||
call::host::{
|
||||
AuthCredential, AuthGetRequest, AuthListRequest, AuthListResult, AuthRuntimeAccount,
|
||||
AuthSaveRequest, AuthSaveResult, CredentialFacts,
|
||||
self as wire, AuthCredential, AuthGetRequest, AuthListRequest, AuthListResult,
|
||||
AuthRuntimeAccount, AuthSaveRequest, AuthSaveResult, CredentialFacts,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -66,6 +66,32 @@ impl PluginAccounts {
|
||||
Self { slot }
|
||||
}
|
||||
|
||||
pub(super) async fn http_proxy(
|
||||
&self,
|
||||
account_id: String,
|
||||
credential_revision: u64,
|
||||
) -> Result<Option<gateway_core::account::OutboundProxy>, PluginFault> {
|
||||
let id = ProviderAccountId::new(account_id).map_err(|_| invalid())?;
|
||||
let expected = Revision::new(credential_revision).map_err(|_| invalid())?;
|
||||
let account = self
|
||||
.slot
|
||||
.upgrade()
|
||||
.map_err(map_admin_error)?
|
||||
.get_runtime(&id)
|
||||
.await
|
||||
.map_err(map_admin_error)?;
|
||||
if !account.enabled {
|
||||
return Err(denied());
|
||||
}
|
||||
if account.credential_revision != expected {
|
||||
return Err(PluginFault::new(
|
||||
ErrorCode::Conflict,
|
||||
"account credential revision has changed",
|
||||
));
|
||||
}
|
||||
Ok(account.effective_outbound_proxy().cloned())
|
||||
}
|
||||
|
||||
pub(super) async fn call(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
@@ -79,7 +105,7 @@ impl PluginAccounts {
|
||||
}
|
||||
let access = self.slot.upgrade().map_err(map_admin_error)?;
|
||||
match method {
|
||||
"host.auth.list" => {
|
||||
wire::AUTH_LIST => {
|
||||
let request: AuthListRequest = decode(payload)?;
|
||||
let limit = PageSize::new(request.limit).map_err(|_| invalid())?;
|
||||
let cursor = request
|
||||
@@ -105,7 +131,7 @@ impl PluginAccounts {
|
||||
next_cursor: page.next_cursor.map(|cursor| cursor.as_str().to_owned()),
|
||||
})
|
||||
}
|
||||
"host.auth.get_runtime" => {
|
||||
wire::AUTH_GET_RUNTIME => {
|
||||
let request: AuthGetRequest = decode(payload)?;
|
||||
let account_id =
|
||||
ProviderAccountId::new(request.account_id).map_err(|_| invalid())?;
|
||||
@@ -115,7 +141,7 @@ impl PluginAccounts {
|
||||
.map_err(map_admin_error)?;
|
||||
encode(&runtime_account(&account))
|
||||
}
|
||||
"host.auth.get" => {
|
||||
wire::AUTH_GET => {
|
||||
let request: AuthGetRequest = decode(payload)?;
|
||||
let account_id =
|
||||
ProviderAccountId::new(request.account_id).map_err(|_| invalid())?;
|
||||
@@ -131,7 +157,7 @@ impl PluginAccounts {
|
||||
facts: credential_facts(account, credential.provider_material),
|
||||
})
|
||||
}
|
||||
"host.auth.save" => {
|
||||
wire::AUTH_SAVE => {
|
||||
let request: AuthSaveRequest = decode(payload)?;
|
||||
encode(&self.save(context, scope, request).await?)
|
||||
}
|
||||
|
||||
@@ -16,9 +16,8 @@ pub(super) struct CallResources {
|
||||
pub(super) cancellation: CancellationToken,
|
||||
pub(super) scope: Arc<CallbackScope>,
|
||||
pub(super) http_resources: Arc<http_middleware::resources::Resources>,
|
||||
pub(super) model_bindings: tokio::sync::Mutex<
|
||||
BTreeMap<String, gateway_core::engine::execution::BoundModelExecutionContext>,
|
||||
>,
|
||||
pub(super) client_identities:
|
||||
tokio::sync::Mutex<BTreeMap<String, gateway_core::engine::execution::AuthenticatedClient>>,
|
||||
pub(super) state: Mutex<CallState>,
|
||||
}
|
||||
|
||||
@@ -32,13 +31,18 @@ impl CallResources {
|
||||
Self {
|
||||
deadline: deadline.clone(),
|
||||
operation_timeout: Duration::from_millis(context.timeout_ms),
|
||||
cancellation: CancellationToken::new(),
|
||||
cancellation: scope
|
||||
.origin
|
||||
.as_ref()
|
||||
.map_or_else(CancellationToken::new, |origin| {
|
||||
origin.cancellation.child_token()
|
||||
}),
|
||||
scope,
|
||||
http_resources: middleware
|
||||
.as_ref()
|
||||
.and_then(|middleware| middleware.http_resources())
|
||||
.unwrap_or_default(),
|
||||
model_bindings: tokio::sync::Mutex::new(BTreeMap::new()),
|
||||
client_identities: tokio::sync::Mutex::new(BTreeMap::new()),
|
||||
state: Mutex::new(CallState {
|
||||
middleware,
|
||||
..CallState::default()
|
||||
@@ -46,6 +50,40 @@ impl CallResources {
|
||||
}
|
||||
}
|
||||
|
||||
/// 子调用共享受信来源;身份解析与执行准入仍留在各能力 owner
|
||||
pub(super) fn child_origin(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
) -> Result<gateway_core::engine::extensions::InvocationOrigin, PluginFault> {
|
||||
let parent = self.scope.origin.as_ref();
|
||||
Ok(gateway_core::engine::extensions::InvocationOrigin {
|
||||
request_id: parent.map_or_else(
|
||||
|| {
|
||||
context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone())
|
||||
},
|
||||
|origin| origin.request_id.clone(),
|
||||
),
|
||||
call_id: parent.map_or_else(
|
||||
|| format!("{}:{}", context.resource_scope_id, context.call_id),
|
||||
|origin| origin.call_id.clone(),
|
||||
),
|
||||
cancellation: self.cancellation.child_token(),
|
||||
extensions: if self.scope.extension_scope.contains(&context.instance_id) {
|
||||
self.scope.extension_scope.clone()
|
||||
} else {
|
||||
self.scope
|
||||
.extension_scope
|
||||
.extending(context.instance_id.clone())
|
||||
.ok_or_else(super::invalid)?
|
||||
},
|
||||
plan: parent.and_then(|origin| origin.plan.clone()),
|
||||
effects: self.scope.execution_effects.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn close(&self) {
|
||||
self.cancellation.cancel();
|
||||
let mut state = self
|
||||
@@ -114,7 +152,9 @@ impl CallResources {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.middleware
|
||||
.clone();
|
||||
middleware.and_then(|middleware| middleware.request_settings())
|
||||
middleware
|
||||
.and_then(|middleware| middleware.request_settings())
|
||||
.or_else(|| self.scope.settings.clone())
|
||||
}
|
||||
|
||||
/// 建立受管流后,网络操作各自计时;连接空闲时间不消耗后续操作的预算
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
//! 客户端认证、冻结权限查询与调用内身份复用
|
||||
|
||||
use std::sync::{Arc, OnceLock, Weak};
|
||||
|
||||
use gateway_admin::model::AdminError;
|
||||
use gateway_core::{
|
||||
account::ProviderAccountId,
|
||||
engine::{
|
||||
authentication::{ClientAuthenticationRequest, ClientIdentityPort},
|
||||
execution::AuthenticatedClient,
|
||||
},
|
||||
policy::ClientApiKeyId,
|
||||
routing::UpstreamModelId,
|
||||
};
|
||||
use gateway_plugin_sdk::{CallContext, ErrorCode, PluginFault, call::data};
|
||||
|
||||
use super::{CallResources, denied, invalid};
|
||||
use crate::RpcReply;
|
||||
|
||||
const MAX_CLIENT_IDENTITIES_PER_CALL: usize = 16;
|
||||
|
||||
pub(crate) struct PluginClientIdentityPortSlot {
|
||||
port: OnceLock<Weak<dyn ClientIdentityPort>>,
|
||||
}
|
||||
|
||||
impl PluginClientIdentityPortSlot {
|
||||
pub(crate) const fn new() -> Self {
|
||||
Self {
|
||||
port: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bind(&self, port: &Arc<dyn ClientIdentityPort>) -> Result<(), AdminError> {
|
||||
self.port
|
||||
.set(Arc::downgrade(port))
|
||||
.map_err(|_| AdminError::conflict("插件身份端口已经绑定"))
|
||||
}
|
||||
|
||||
fn upgrade(&self) -> Result<Arc<dyn ClientIdentityPort>, PluginFault> {
|
||||
self.port.get().and_then(Weak::upgrade).ok_or_else(denied)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PluginClients {
|
||||
slot: Arc<PluginClientIdentityPortSlot>,
|
||||
}
|
||||
|
||||
impl PluginClients {
|
||||
pub(super) fn new(slot: Arc<PluginClientIdentityPortSlot>) -> Self {
|
||||
Self { slot }
|
||||
}
|
||||
|
||||
pub(super) async fn call(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
call: &Arc<CallResources>,
|
||||
method: &str,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
match method {
|
||||
data::CLIENT_VERIFY => {
|
||||
if params != serde_json::json!({}) {
|
||||
return Err(invalid());
|
||||
}
|
||||
let request: data::VerifyClientRequest =
|
||||
serde_json::from_slice(&payload).map_err(|_| invalid())?;
|
||||
let mut authentication = ClientAuthenticationRequest::new(request.authorization)
|
||||
.map_err(|_| invalid())?
|
||||
.with_invocation(call.child_origin(context)?);
|
||||
if let Some(settings) = call.request_settings() {
|
||||
authentication = authentication.with_settings(settings);
|
||||
}
|
||||
let port = self.slot.upgrade()?;
|
||||
call.observe_dispatch()?;
|
||||
let client = port
|
||||
.verify_client(authentication)
|
||||
.await
|
||||
.map_err(super::error::gateway)?;
|
||||
super::admin::encode(&data::VerifiedClient {
|
||||
client_key_id: client.policy().key_id().as_str().to_owned(),
|
||||
})
|
||||
}
|
||||
data::ACCESS_CHECK => {
|
||||
if params != serde_json::json!({}) {
|
||||
return Err(invalid());
|
||||
}
|
||||
let request: data::AccessQuery =
|
||||
serde_json::from_slice(&payload).map_err(|_| invalid())?;
|
||||
let client = self.identity(call, request.client_key_id).await?;
|
||||
let scope = client.policy().account_scope();
|
||||
let model = request
|
||||
.upstream_model
|
||||
.map(UpstreamModelId::new)
|
||||
.transpose()
|
||||
.map_err(|_| invalid())?;
|
||||
let accounts = request
|
||||
.account_ids
|
||||
.into_iter()
|
||||
.map(|account_id| {
|
||||
let account =
|
||||
ProviderAccountId::new(account_id.clone()).map_err(|_| invalid())?;
|
||||
let allowed = model.as_ref().map_or_else(
|
||||
|| scope.allows(&account),
|
||||
|model| scope.allows_model(&account, model.as_str()),
|
||||
);
|
||||
Ok(data::AccountAccessFacts {
|
||||
account_id,
|
||||
allowed,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, PluginFault>>()?;
|
||||
super::admin::encode(&data::AccessFacts {
|
||||
schema_version: 1,
|
||||
client_key_id: client.policy().key_id().as_str().to_owned(),
|
||||
accounts,
|
||||
})
|
||||
}
|
||||
_ => Err(denied()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn identity(
|
||||
&self,
|
||||
call: &CallResources,
|
||||
client_key_id: String,
|
||||
) -> Result<AuthenticatedClient, PluginFault> {
|
||||
let port = self.slot.upgrade()?;
|
||||
let key = ClientApiKeyId::new(client_key_id.clone()).map_err(|_| invalid())?;
|
||||
let settings = call.request_settings();
|
||||
let mut identities = call.client_identities.lock().await;
|
||||
if let Some(client) = identities
|
||||
.get(&client_key_id)
|
||||
.filter(|client| client.request_settings() == settings.as_ref())
|
||||
{
|
||||
return Ok(client.clone());
|
||||
}
|
||||
if !identities.contains_key(&client_key_id)
|
||||
&& identities.len() >= MAX_CLIENT_IDENTITIES_PER_CALL
|
||||
{
|
||||
return Err(PluginFault::new(
|
||||
ErrorCode::Capacity,
|
||||
"client identity capacity is exhausted",
|
||||
));
|
||||
}
|
||||
let client = port
|
||||
.resolve_client(key, settings)
|
||||
.await
|
||||
.map_err(super::error::gateway)?;
|
||||
// 查询与执行共用调用内快照,执行链和子执行取消只在真正启动时建立
|
||||
identities.insert(client_key_id, client.clone());
|
||||
Ok(client)
|
||||
}
|
||||
}
|
||||
@@ -98,7 +98,7 @@ impl PluginData {
|
||||
next_cursor: page.next_cursor.map(|id| id.as_str().to_owned()),
|
||||
})
|
||||
}
|
||||
data::QUOTA_GET | data::QUOTA_REFRESH => {
|
||||
data::QUOTA_GET | data::QUOTA_REFRESH | data::QUOTA_DETAILS => {
|
||||
let accounts = self.accounts.upgrade().map_err(map_admin_error)?;
|
||||
let query: data::QuotaFactsQuery =
|
||||
serde_json::from_slice(payload).map_err(|_| invalid())?;
|
||||
@@ -109,6 +109,35 @@ impl PluginData {
|
||||
accounts.get_quota(&account_id).await
|
||||
}
|
||||
.map_err(map_admin_error)?;
|
||||
if method == data::QUOTA_DETAILS {
|
||||
return super::admin::encode(&data::QuotaDetails {
|
||||
schema_version: 1,
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
observed_at_ms: quota.observed_at.map(|time| time.timestamp_millis()),
|
||||
limit_reached: quota.limit_reached,
|
||||
windows: quota
|
||||
.windows
|
||||
.into_iter()
|
||||
.map(|window| data::QuotaWindowDetails {
|
||||
key: window.key,
|
||||
group: window.group,
|
||||
label: window.label,
|
||||
limit_id: window.limit_id,
|
||||
limit_name: window.limit_name,
|
||||
role: window.role.map(|role| role.as_str().to_owned()),
|
||||
window_seconds: window.window_seconds,
|
||||
used_percent: window.used_percent,
|
||||
reset_at_ms: window.reset_at.map(|time| time.timestamp_millis()),
|
||||
limit_reached: window.limit_reached,
|
||||
})
|
||||
.collect(),
|
||||
credits: quota.credits.map(|credits| data::QuotaCredits {
|
||||
has_credits: credits.has_credits,
|
||||
unlimited: credits.unlimited,
|
||||
balance: credits.balance,
|
||||
}),
|
||||
});
|
||||
}
|
||||
serde_json::to_vec(&data::QuotaFacts {
|
||||
schema_version: 1,
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
//! 将有效父调用的宿主回调分派到所属原生能力,并随调用撤销资源
|
||||
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_plugin_sdk::{
|
||||
CallContext, ErrorCode, PluginFault,
|
||||
call::{host, middleware as middleware_wire, upstream_adapter as upstream_wire},
|
||||
client::CallDeadline,
|
||||
};
|
||||
|
||||
use super::{CallResources, CallbackScope, PluginCallbacks, denied, http, http_dispatch, upstream};
|
||||
use crate::{CallbackHandler, RpcReply};
|
||||
|
||||
impl CallbackHandler for PluginCallbacks {
|
||||
fn begin(&self, context: &CallContext, deadline: &CallDeadline) {
|
||||
let scope = self
|
||||
.scopes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&context.resource_scope_id)
|
||||
.and_then(Weak::upgrade)
|
||||
.unwrap_or_else(|| {
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
if context.stage == gateway_plugin_sdk::Stage::Registration {
|
||||
scope.request_body_limit = self.registration_body_limit;
|
||||
}
|
||||
Arc::new(scope)
|
||||
});
|
||||
let middleware = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&context.resource_scope_id);
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
context.call_id,
|
||||
Arc::new(CallResources::new(context, deadline, scope, middleware)),
|
||||
);
|
||||
}
|
||||
|
||||
fn finish(&self, context: &CallContext) {
|
||||
if let Some(call) = self
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&context.call_id)
|
||||
{
|
||||
call.close();
|
||||
}
|
||||
}
|
||||
|
||||
fn call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
method: String,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> BoxFuture<'static, Result<RpcReply, PluginFault>> {
|
||||
let call = self
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&context.call_id)
|
||||
.cloned();
|
||||
let http = self.http.clone();
|
||||
let network = self.network.clone();
|
||||
let maximum_payload = self.maximum_payload;
|
||||
let log = self.log.clone();
|
||||
let private_state = self.private_state.clone();
|
||||
let accounts = self.accounts.clone();
|
||||
let resources = self.resources.clone();
|
||||
let data = self.data.clone();
|
||||
let keys = self.keys.clone();
|
||||
let clients = self.clients.clone();
|
||||
let models = self.models.clone();
|
||||
let request_limits = self.request_limits.clone();
|
||||
let affinity = self.affinity.clone();
|
||||
let services = self.services.clone();
|
||||
let service_actor = self.service_actor.clone();
|
||||
Box::pin(async move {
|
||||
let call = call.ok_or_else(denied)?;
|
||||
let scope = &call.scope;
|
||||
let (middleware, is_private_state) = {
|
||||
let state = call
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.closed || call.deadline.is_elapsed() {
|
||||
return Err(denied());
|
||||
}
|
||||
if method == host::LOG {
|
||||
return log.record(&context, params, &payload);
|
||||
}
|
||||
let middleware = if matches!(
|
||||
method.as_str(),
|
||||
middleware_wire::NEXT_METHOD
|
||||
| middleware_wire::BODY_READ_METHOD
|
||||
| middleware_wire::BODY_FACTS_METHOD
|
||||
| middleware_wire::BODY_CLOSE_METHOD
|
||||
| middleware_wire::http::UPGRADE_METHOD
|
||||
| middleware_wire::http::RECEIVE_METHOD
|
||||
| middleware_wire::websocket::SEND_METHOD
|
||||
) {
|
||||
Some(state.middleware.clone().ok_or_else(denied)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let is_private_state = matches!(
|
||||
method.as_str(),
|
||||
host::STATE_GET | host::STATE_PUT | host::STATE_DELETE
|
||||
);
|
||||
(middleware, is_private_state)
|
||||
};
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
middleware_wire::http::DISPATCH_METHOD
|
||||
| middleware_wire::http::BODY_READ_METHOD
|
||||
| middleware_wire::http::BODY_CLOSE_METHOD
|
||||
| middleware_wire::http::BODY_CREATE_METHOD
|
||||
| middleware_wire::http::BODY_WRITE_METHOD
|
||||
) {
|
||||
return tokio::select! {
|
||||
biased;
|
||||
() = call.cancellation.cancelled() => Err(PluginFault::new(ErrorCode::Cancelled, "HTTP callback cancelled")),
|
||||
result = http_dispatch::call(&services, &context, &call, &method, params, payload, maximum_payload) => result,
|
||||
};
|
||||
}
|
||||
if let Some(middleware) = middleware {
|
||||
return middleware
|
||||
.invoke(method, params, payload, maximum_payload)
|
||||
.await;
|
||||
}
|
||||
if method == gateway_plugin_sdk::call::services::CALL_METHOD {
|
||||
return services
|
||||
.call(&context, &call, service_actor, params, &payload)
|
||||
.await;
|
||||
}
|
||||
if is_private_state {
|
||||
return private_state.call(&method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::resources::GROUP_ENSURE
|
||||
| gateway_plugin_sdk::call::resources::GROUP_MEMBERS
|
||||
| gateway_plugin_sdk::call::resources::KEY_ENSURE
|
||||
) {
|
||||
return resources.call(&context, &method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
host::KEYS_LIST
|
||||
| gateway_plugin_sdk::call::key_budgets::RESET
|
||||
| gateway_plugin_sdk::call::key_budgets::GET
|
||||
| gateway_plugin_sdk::call::key_budgets::UPDATE_LIMITS
|
||||
) {
|
||||
return keys.call(&context, &method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::data::CLIENT_VERIFY
|
||||
| gateway_plugin_sdk::call::data::ACCESS_CHECK
|
||||
) {
|
||||
return clients
|
||||
.call(&context, &call, &method, params, payload)
|
||||
.await;
|
||||
}
|
||||
if method == host::MODELS_LIST {
|
||||
return models.call(&context, &call, &method, params, payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
host::MODEL_EXECUTE
|
||||
| host::MODEL_EXECUTE_STREAM
|
||||
| host::MODEL_STREAM_READ
|
||||
| host::MODEL_STREAM_CLOSE
|
||||
) {
|
||||
return models.call(&context, &call, &method, params, payload).await;
|
||||
}
|
||||
if method == host::AFFINITY_LOOKUP {
|
||||
return affinity.call(&context, params, &payload).await;
|
||||
}
|
||||
let scope = Some(scope)
|
||||
.filter(|scope| scope.authorizes(&context))
|
||||
.ok_or_else(denied)?;
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::data::ACCOUNTS_LIST
|
||||
| gateway_plugin_sdk::call::data::KEYS_GET
|
||||
| gateway_plugin_sdk::call::data::QUOTA_GET
|
||||
| gateway_plugin_sdk::call::data::QUOTA_REFRESH
|
||||
| gateway_plugin_sdk::call::data::QUOTA_DETAILS
|
||||
) {
|
||||
return data.call(&method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
host::AUTH_LIST | host::AUTH_GET | host::AUTH_GET_RUNTIME | host::AUTH_SAVE
|
||||
) {
|
||||
return accounts
|
||||
.call(&context, scope, &method, params, &payload)
|
||||
.await;
|
||||
}
|
||||
let http = http::HttpCallbacks {
|
||||
client: &http,
|
||||
network: &network,
|
||||
scope,
|
||||
call: &call,
|
||||
maximum_payload,
|
||||
request_limits: &request_limits,
|
||||
};
|
||||
if method == host::ACCOUNT_HTTP {
|
||||
return http.account(&accounts, params, payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
upstream_wire::HTTP_DO
|
||||
| upstream_wire::HTTP_DO_STREAM
|
||||
| upstream_wire::HTTP_STREAM_READ
|
||||
| upstream_wire::HTTP_STREAM_CLOSE
|
||||
| upstream_wire::WEBSOCKET_OPEN
|
||||
| upstream_wire::WEBSOCKET_SEND
|
||||
| upstream_wire::WEBSOCKET_READ
|
||||
| upstream_wire::WEBSOCKET_CLOSE
|
||||
) {
|
||||
return upstream::dispatch(&http, &method, params, payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
host::HTTP_DO
|
||||
| host::HTTP_DO_STREAM
|
||||
| host::HTTP_STREAM_READ
|
||||
| host::HTTP_STREAM_CLOSE
|
||||
) {
|
||||
return http.dispatch(&method, params, payload).await;
|
||||
}
|
||||
Err(PluginFault::new(
|
||||
ErrorCode::Unsupported,
|
||||
"host callback method is not supported",
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -29,16 +29,42 @@ impl HttpCallbacks<'_> {
|
||||
self.request_limits.body_bytes(self.call)
|
||||
}
|
||||
|
||||
pub(super) async fn account(
|
||||
&self,
|
||||
accounts: &super::accounts::PluginAccounts,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
let request: wire::AccountHttpRequest = decode(params)?;
|
||||
let maximum_body_bytes = self.body_limit(&payload)?;
|
||||
let proxy = accounts
|
||||
.http_proxy(request.account_id, request.credential_revision)
|
||||
.await?;
|
||||
self.send(
|
||||
HttpRequest {
|
||||
method: request.request.method,
|
||||
url: request.request.url,
|
||||
headers: request.request.headers,
|
||||
body: payload,
|
||||
maximum_body_bytes,
|
||||
},
|
||||
true,
|
||||
None,
|
||||
proxy.as_ref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn send(
|
||||
&self,
|
||||
request: HttpRequest,
|
||||
streaming: bool,
|
||||
upstream_purpose: Option<gateway_plugin_sdk::call::upstream_adapter::UpstreamPathPurpose>,
|
||||
proxy: Option<&gateway_core::account::OutboundProxy>,
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
let Self {
|
||||
client,
|
||||
network,
|
||||
scope,
|
||||
call,
|
||||
maximum_payload,
|
||||
..
|
||||
@@ -48,9 +74,7 @@ impl HttpCallbacks<'_> {
|
||||
}
|
||||
let timeout = call.timeout()?;
|
||||
let attempt = call.start_upstream(upstream_purpose)?;
|
||||
let response = client
|
||||
.open(request, scope.proxy.as_ref(), network, timeout)
|
||||
.await;
|
||||
let response = client.open(request, proxy, network, timeout).await;
|
||||
attempt.finish(
|
||||
response
|
||||
.as_ref()
|
||||
@@ -115,7 +139,7 @@ impl HttpCallbacks<'_> {
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
let call = self.call;
|
||||
match method {
|
||||
"host.http.do" | "host.http.do_stream" => {
|
||||
wire::HTTP_DO | wire::HTTP_DO_STREAM => {
|
||||
let request: wire::HttpRequest = decode(params)?;
|
||||
self.send(
|
||||
HttpRequest {
|
||||
@@ -125,12 +149,13 @@ impl HttpCallbacks<'_> {
|
||||
maximum_body_bytes: self.body_limit(&payload)?,
|
||||
body: payload,
|
||||
},
|
||||
method == "host.http.do_stream",
|
||||
method == wire::HTTP_DO_STREAM,
|
||||
None,
|
||||
self.scope.proxy.as_ref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
"host.http.stream_read" => {
|
||||
wire::HTTP_STREAM_READ => {
|
||||
if !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -174,7 +199,7 @@ impl HttpCallbacks<'_> {
|
||||
payload: chunk.map_or_else(Vec::new, |chunk| chunk.to_vec()),
|
||||
})
|
||||
}
|
||||
"host.http.stream_close" => {
|
||||
wire::HTTP_STREAM_CLOSE => {
|
||||
if !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
|
||||
@@ -36,25 +36,23 @@ pub(super) async fn call(
|
||||
request.timeout_ms,
|
||||
&context.instance_id,
|
||||
)?;
|
||||
let parent = call.scope.origin.as_ref();
|
||||
call.scope.check_child_depth(&context.instance_id)?;
|
||||
let origin = call.child_origin(context)?;
|
||||
let child = core::Context {
|
||||
plugin_instance_id: Some(context.instance_id.clone()),
|
||||
request_id: context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone()),
|
||||
request_id: origin.request_id,
|
||||
call_id: uuid::Uuid::now_v7().to_string(),
|
||||
parent_call_id: Some(parent.map_or_else(
|
||||
|| format!("{}:{}", context.resource_scope_id, context.call_id),
|
||||
|parent| parent.call_id.clone(),
|
||||
)),
|
||||
extensions: call.scope.child_extensions(&context.instance_id)?,
|
||||
// 句柄可以在父 RPC End 后交给 HTTP 发送端;生命周期属于父调用和响应正文,不能在 RPC End 提前取消
|
||||
cancellation: parent.map_or_else(
|
||||
|| call.cancellation.child_token(),
|
||||
|parent| parent.cancellation.child_token(),
|
||||
),
|
||||
plan: parent.map(|parent| parent.plan.clone()),
|
||||
parent_call_id: Some(origin.call_id),
|
||||
extensions: origin.extensions,
|
||||
// 转交正文后由消费端持有原请求寿命,父 RPC End 只撤销新业务分派
|
||||
cancellation: call
|
||||
.scope
|
||||
.origin
|
||||
.as_ref()
|
||||
.map_or(origin.cancellation, |parent| {
|
||||
parent.cancellation.child_token()
|
||||
}),
|
||||
plan: origin.plan,
|
||||
};
|
||||
let body = call.http_resources.body(request.body, payload).await?;
|
||||
let mut http = core::Request::new(body);
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
//! 为具体入口登记回调作用域和一次性中间件绑定,不持有 RPC 调度状态
|
||||
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use gateway_core::{account::OutboundProxy, lifecycle::CancellationToken};
|
||||
use gateway_plugin_sdk::{CallContext, PluginFault};
|
||||
|
||||
use super::{
|
||||
CallbackScope, InvocationContext, MiddlewareBinding, MiddlewareCallback, PluginCallbacks,
|
||||
denied, upstream,
|
||||
};
|
||||
|
||||
impl PluginCallbacks {
|
||||
pub(crate) fn bind_middleware(
|
||||
self: &Arc<Self>,
|
||||
resource_scope_id: String,
|
||||
invocation: Arc<dyn MiddlewareCallback>,
|
||||
scope: Arc<CallbackScope>,
|
||||
) -> Result<MiddlewareBinding, gateway_core::engine::middleware::MiddlewareError> {
|
||||
let mut pending = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if pending
|
||||
.insert(resource_scope_id.clone(), Arc::clone(&invocation))
|
||||
.is_some()
|
||||
{
|
||||
return Err(gateway_core::engine::middleware::MiddlewareError::InvalidState);
|
||||
}
|
||||
Ok(MiddlewareBinding::new(
|
||||
Arc::clone(self),
|
||||
resource_scope_id,
|
||||
invocation,
|
||||
scope,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn unbind_middleware(
|
||||
&self,
|
||||
resource_scope_id: &str,
|
||||
invocation: &Arc<dyn MiddlewareCallback>,
|
||||
) {
|
||||
let mut pending = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if pending
|
||||
.get(resource_scope_id)
|
||||
.is_some_and(|pending| Arc::ptr_eq(pending, invocation))
|
||||
{
|
||||
pending.remove(resource_scope_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_data_plane(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
effects: Option<Arc<gateway_core::engine::nested::ExecutionEffects>>,
|
||||
extensions: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
plan: Option<gateway_core::engine::middleware::FrozenMiddlewarePlan>,
|
||||
cancellation: CancellationToken,
|
||||
request_body_limit: Option<usize>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if !matches!(
|
||||
context.stage,
|
||||
gateway_plugin_sdk::Stage::Request
|
||||
| gateway_plugin_sdk::Stage::Attempt
|
||||
| gateway_plugin_sdk::Stage::Outbound
|
||||
| gateway_plugin_sdk::Stage::Routing
|
||||
| gateway_plugin_sdk::Stage::Scheduling
|
||||
| gateway_plugin_sdk::Stage::Retry
|
||||
) {
|
||||
return Err(denied());
|
||||
}
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
if let Some(effects) = effects {
|
||||
scope = scope.with_execution_effects(effects);
|
||||
}
|
||||
scope.extension_scope = extensions;
|
||||
scope.request_body_limit = request_body_limit;
|
||||
scope.origin = plan.map(|plan| InvocationContext {
|
||||
request_id: context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone()),
|
||||
call_id: context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone()),
|
||||
cancellation,
|
||||
extensions: scope.extension_scope.clone(),
|
||||
effects: scope.execution_effects.clone(),
|
||||
plan: Some(plan),
|
||||
});
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn request_body_limit(&self) -> Result<usize, PluginFault> {
|
||||
self.request_limits.current_body_bytes()
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_upstream(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
managed: Arc<upstream::ManagedUpstream>,
|
||||
extension_scope: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
request_body_limit: usize,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::Upstream
|
||||
|| context.account_id.as_deref() != Some(managed.account.account_id().as_str())
|
||||
|| context.credential_revision != Some(managed.account.credential_revision().get())
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
let mut scope = CallbackScope::new(context, managed.account.outbound_proxy().cloned())
|
||||
.with_upstream(managed);
|
||||
scope.extension_scope = extension_scope;
|
||||
scope.request_body_limit = Some(request_body_limit);
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_management(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
proxy: Option<OutboundProxy>,
|
||||
request: Option<Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if !matches!(
|
||||
context.stage,
|
||||
gateway_plugin_sdk::Stage::Management | gateway_plugin_sdk::Stage::PublicManagement
|
||||
) {
|
||||
return Err(denied());
|
||||
}
|
||||
self.prepare_scope(context, proxy, request)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_command_line(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::CommandLine {
|
||||
return Err(denied());
|
||||
}
|
||||
self.prepare_scope(context, None, None)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_frontend_authentication(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
request: &gateway_core::engine::authentication::ClientAuthenticationRequest,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::Authentication
|
||||
|| context.account_id.is_some()
|
||||
|| context.credential_revision.is_some()
|
||||
|| context.attempt_id.is_some()
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
scope.settings = request.settings().cloned();
|
||||
if let Some(origin) = request.invocation() {
|
||||
scope.extension_scope = origin.extensions.clone();
|
||||
scope.execution_effects = origin.effects.clone();
|
||||
scope.origin = Some(origin.clone());
|
||||
} else if let Some(request) = request.shared_request_context() {
|
||||
scope.extension_scope = scope.extension_scope.with_request(request);
|
||||
}
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_command_account(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
scope: &CallbackScope,
|
||||
request: gateway_plugin_sdk::call::host::AuthSaveRequest,
|
||||
) -> Result<gateway_plugin_sdk::call::host::AuthSaveResult, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::CommandLine || !scope.authorizes(context) {
|
||||
return Err(denied());
|
||||
}
|
||||
self.accounts.save(context, scope, request).await
|
||||
}
|
||||
|
||||
fn prepare_scope(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
proxy: Option<OutboundProxy>,
|
||||
request: Option<Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let mut scope = CallbackScope::new(context, proxy);
|
||||
if let Some(request) = request {
|
||||
scope.extension_scope = scope.extension_scope.with_request(request);
|
||||
}
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_extensions(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
extension_scope: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
scope.extension_scope = extension_scope;
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn bind_invocation(
|
||||
self: &Arc<Self>,
|
||||
call: &CallContext,
|
||||
origin: Option<InvocationContext>,
|
||||
extensions: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
invocation: Arc<dyn MiddlewareCallback>,
|
||||
) -> Result<MiddlewareBinding, gateway_core::engine::middleware::MiddlewareError> {
|
||||
let mut scope = CallbackScope::for_call(call);
|
||||
scope.extension_scope = extensions;
|
||||
scope.origin = origin;
|
||||
let scope = self
|
||||
.register_scope(call, scope)
|
||||
.map_err(|_| gateway_core::engine::middleware::MiddlewareError::Fault)?;
|
||||
self.bind_middleware(call.resource_scope_id.clone(), invocation, scope)
|
||||
}
|
||||
|
||||
fn register_scope(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
scope: CallbackScope,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let scope = Arc::new(scope);
|
||||
let mut scopes = self
|
||||
.scopes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
scopes.retain(|_, scope| scope.strong_count() != 0);
|
||||
if scopes
|
||||
.get(&context.resource_scope_id)
|
||||
.and_then(Weak::upgrade)
|
||||
.is_some()
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
scopes.insert(context.resource_scope_id.clone(), Arc::downgrade(&scope));
|
||||
Ok(scope)
|
||||
}
|
||||
}
|
||||
@@ -52,7 +52,7 @@ impl PluginClientKeys {
|
||||
params: serde_json::Value,
|
||||
payload: &[u8],
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
if method == "host.keys.list" {
|
||||
if method == gateway_plugin_sdk::call::host::KEYS_LIST {
|
||||
return self.ports.list(params, payload).await;
|
||||
}
|
||||
if params != serde_json::json!({}) {
|
||||
|
||||
@@ -1,20 +1,24 @@
|
||||
//! 插件宿主回调端口的组合、授权检查与方法分派
|
||||
//! 插件宿主回调端口、调用作用域与分派的模块入口
|
||||
|
||||
mod accounts;
|
||||
mod admin;
|
||||
mod affinity;
|
||||
mod call_resources;
|
||||
mod client;
|
||||
mod data;
|
||||
mod dispatch;
|
||||
pub(crate) mod error;
|
||||
mod facts;
|
||||
mod http;
|
||||
mod http_dispatch;
|
||||
pub(crate) mod http_middleware;
|
||||
mod invocation;
|
||||
mod keys;
|
||||
mod log;
|
||||
mod middleware;
|
||||
mod model;
|
||||
pub(crate) mod outbound;
|
||||
mod ports;
|
||||
pub(crate) mod private_state;
|
||||
pub(crate) mod request_limits;
|
||||
mod resources;
|
||||
@@ -23,21 +27,11 @@ pub(crate) mod services;
|
||||
pub(crate) mod upstream;
|
||||
pub(crate) mod websocket_middleware;
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex, Weak},
|
||||
};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::model::AdminError;
|
||||
use gateway_core::{account::OutboundProxy, lifecycle::CancellationToken};
|
||||
use gateway_host::outbound::{HttpClient, NetworkPolicy};
|
||||
use gateway_plugin_sdk::{CallContext, ErrorCode, PluginFault, client::CallDeadline};
|
||||
|
||||
use crate::{CallbackHandler, RpcReply};
|
||||
pub(crate) use accounts::PluginAccountPortSlot;
|
||||
pub(crate) use affinity::PluginAffinityPortSlot;
|
||||
use call_resources::{CallResources, HttpStream};
|
||||
pub(crate) use client::PluginClientIdentityPortSlot;
|
||||
use gateway_plugin_sdk::{ErrorCode, PluginFault};
|
||||
pub(crate) use keys::PluginClientKeyPortSlot;
|
||||
pub(crate) use middleware::{
|
||||
MiddlewareBinding, MiddlewareBodyAuthority, MiddlewareCallback, MiddlewareCompletionBody,
|
||||
@@ -47,509 +41,7 @@ pub(crate) use model::PluginModelPortSlot;
|
||||
pub(crate) use resources::PluginResourcePorts;
|
||||
pub(crate) use scope::{CallbackScope, InvocationContext};
|
||||
|
||||
pub(crate) struct PluginCallbackPorts {
|
||||
services: Arc<services::ServicePorts>,
|
||||
http: Arc<HttpClient>,
|
||||
network: NetworkPolicy,
|
||||
accounts: Arc<PluginAccountPortSlot>,
|
||||
keys: Arc<PluginClientKeyPortSlot>,
|
||||
models: Arc<PluginModelPortSlot>,
|
||||
affinity: Arc<PluginAffinityPortSlot>,
|
||||
resources: Arc<PluginResourcePorts>,
|
||||
}
|
||||
|
||||
impl PluginCallbackPorts {
|
||||
pub(crate) fn new(
|
||||
services: Arc<services::ServicePorts>,
|
||||
http: Arc<HttpClient>,
|
||||
accounts: Arc<PluginAccountPortSlot>,
|
||||
keys: Arc<PluginClientKeyPortSlot>,
|
||||
models: Arc<PluginModelPortSlot>,
|
||||
affinity: Arc<PluginAffinityPortSlot>,
|
||||
resources: Arc<PluginResourcePorts>,
|
||||
) -> Self {
|
||||
Self {
|
||||
services,
|
||||
http,
|
||||
network: NetworkPolicy::unrestricted(),
|
||||
accounts,
|
||||
keys,
|
||||
models,
|
||||
affinity,
|
||||
resources,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct PluginCallbacks {
|
||||
pub(crate) registration_body_limit: Option<usize>,
|
||||
services: Arc<services::ServicePorts>,
|
||||
service_actor: gateway_admin::model::usage_data::UsageActor,
|
||||
accounts: Arc<accounts::PluginAccounts>,
|
||||
resources: Arc<resources::PluginResources>,
|
||||
data: Arc<data::PluginData>,
|
||||
keys: Arc<keys::PluginClientKeys>,
|
||||
affinity: Arc<affinity::PluginAffinity>,
|
||||
log: Arc<log::PluginLog>,
|
||||
models: Arc<model::PluginModels>,
|
||||
private_state: Arc<private_state::PluginPrivateState>,
|
||||
http: Arc<HttpClient>,
|
||||
network: Arc<NetworkPolicy>,
|
||||
scopes: Mutex<BTreeMap<String, Weak<CallbackScope>>>,
|
||||
pending_middleware: Mutex<BTreeMap<String, Arc<dyn MiddlewareCallback>>>,
|
||||
calls: Mutex<BTreeMap<u64, Arc<CallResources>>>,
|
||||
maximum_payload: usize,
|
||||
request_limits: Arc<request_limits::RequestLimits>,
|
||||
}
|
||||
|
||||
impl PluginCallbacks {
|
||||
pub(crate) fn new(
|
||||
instance: &gateway_admin::model::plugins::instances::PluginInstance,
|
||||
maximum_payload: usize,
|
||||
manifest: &gateway_plugin_sdk::Manifest,
|
||||
log_slots: Arc<tokio::sync::Semaphore>,
|
||||
private_state: Arc<private_state::PluginPrivateState>,
|
||||
ports: PluginCallbackPorts,
|
||||
request_limits: Arc<request_limits::RequestLimits>,
|
||||
) -> Result<Self, AdminError> {
|
||||
Ok(Self {
|
||||
registration_body_limit: None,
|
||||
services: ports.services,
|
||||
service_actor: gateway_admin::model::usage_data::UsageActor::Plugin {
|
||||
publisher: manifest.publisher.clone(),
|
||||
name: manifest.name.clone(),
|
||||
instance_id: instance.id.clone(),
|
||||
},
|
||||
resources: Arc::new(resources::PluginResources::new(instance, ports.resources)),
|
||||
data: Arc::new(data::PluginData::new(
|
||||
ports.accounts.clone(),
|
||||
ports.keys.clone(),
|
||||
)),
|
||||
accounts: Arc::new(accounts::PluginAccounts::new(ports.accounts)),
|
||||
keys: Arc::new(keys::PluginClientKeys::new(instance, ports.keys)),
|
||||
affinity: Arc::new(affinity::PluginAffinity::new(ports.affinity)),
|
||||
log: Arc::new(
|
||||
log::PluginLog::new(manifest, log_slots)
|
||||
.map_err(|_| AdminError::invalid("插件清单身份不合法"))?,
|
||||
),
|
||||
models: Arc::new(model::PluginModels::new(
|
||||
ports.models,
|
||||
request_limits.clone(),
|
||||
maximum_payload,
|
||||
)),
|
||||
private_state,
|
||||
http: ports.http,
|
||||
network: Arc::new(ports.network),
|
||||
scopes: Mutex::new(BTreeMap::new()),
|
||||
pending_middleware: Mutex::new(BTreeMap::new()),
|
||||
calls: Mutex::new(BTreeMap::new()),
|
||||
maximum_payload,
|
||||
request_limits,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn bind_middleware(
|
||||
self: &Arc<Self>,
|
||||
resource_scope_id: String,
|
||||
invocation: Arc<dyn MiddlewareCallback>,
|
||||
scope: Arc<CallbackScope>,
|
||||
) -> Result<MiddlewareBinding, gateway_core::engine::middleware::MiddlewareError> {
|
||||
let mut pending = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if pending
|
||||
.insert(resource_scope_id.clone(), Arc::clone(&invocation))
|
||||
.is_some()
|
||||
{
|
||||
return Err(gateway_core::engine::middleware::MiddlewareError::InvalidState);
|
||||
}
|
||||
Ok(MiddlewareBinding::new(
|
||||
Arc::clone(self),
|
||||
resource_scope_id,
|
||||
invocation,
|
||||
scope,
|
||||
))
|
||||
}
|
||||
|
||||
fn unbind_middleware(&self, resource_scope_id: &str, invocation: &Arc<dyn MiddlewareCallback>) {
|
||||
let mut pending = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if pending
|
||||
.get(resource_scope_id)
|
||||
.is_some_and(|pending| Arc::ptr_eq(pending, invocation))
|
||||
{
|
||||
pending.remove(resource_scope_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_data_plane(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
effects: Option<Arc<gateway_core::engine::nested::ExecutionEffects>>,
|
||||
extensions: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
plan: Option<gateway_core::engine::middleware::FrozenMiddlewarePlan>,
|
||||
cancellation: CancellationToken,
|
||||
request_body_limit: Option<usize>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if !matches!(
|
||||
context.stage,
|
||||
gateway_plugin_sdk::Stage::Request
|
||||
| gateway_plugin_sdk::Stage::Attempt
|
||||
| gateway_plugin_sdk::Stage::Outbound
|
||||
| gateway_plugin_sdk::Stage::Routing
|
||||
| gateway_plugin_sdk::Stage::Scheduling
|
||||
| gateway_plugin_sdk::Stage::Retry
|
||||
) {
|
||||
return Err(denied());
|
||||
}
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
if let Some(effects) = effects {
|
||||
scope = scope.with_execution_effects(effects);
|
||||
}
|
||||
scope.extension_scope = extensions;
|
||||
scope.request_body_limit = request_body_limit;
|
||||
scope.origin = plan.map(|plan| InvocationContext {
|
||||
request_id: context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone()),
|
||||
call_id: context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone()),
|
||||
cancellation,
|
||||
plan,
|
||||
});
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn request_body_limit(&self) -> Result<usize, PluginFault> {
|
||||
self.request_limits.current_body_bytes()
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_upstream(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
managed: Arc<upstream::ManagedUpstream>,
|
||||
extension_scope: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
request_body_limit: usize,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::Upstream
|
||||
|| context.account_id.as_deref() != Some(managed.account.account_id().as_str())
|
||||
|| context.credential_revision != Some(managed.account.credential_revision().get())
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
let mut scope = CallbackScope::new(context, managed.account.outbound_proxy().cloned())
|
||||
.with_upstream(managed);
|
||||
scope.extension_scope = extension_scope;
|
||||
scope.request_body_limit = Some(request_body_limit);
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_management(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
proxy: Option<OutboundProxy>,
|
||||
request: Option<Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if !matches!(
|
||||
context.stage,
|
||||
gateway_plugin_sdk::Stage::Management | gateway_plugin_sdk::Stage::PublicManagement
|
||||
) {
|
||||
return Err(denied());
|
||||
}
|
||||
self.prepare_scope(context, proxy, request)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_command_line(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::CommandLine {
|
||||
return Err(denied());
|
||||
}
|
||||
self.prepare_scope(context, None, None)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_frontend_authentication(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
request: Option<Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::Authentication
|
||||
|| context.account_id.is_some()
|
||||
|| context.credential_revision.is_some()
|
||||
|| context.attempt_id.is_some()
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
self.prepare_scope(context, None, request)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_command_account(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
scope: &CallbackScope,
|
||||
request: gateway_plugin_sdk::call::host::AuthSaveRequest,
|
||||
) -> Result<gateway_plugin_sdk::call::host::AuthSaveResult, PluginFault> {
|
||||
if context.stage != gateway_plugin_sdk::Stage::CommandLine || !scope.authorizes(context) {
|
||||
return Err(denied());
|
||||
}
|
||||
self.accounts.save(context, scope, request).await
|
||||
}
|
||||
|
||||
fn prepare_scope(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
proxy: Option<OutboundProxy>,
|
||||
request: Option<Arc<gateway_core::engine::extensions::RequestContext>>,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let mut scope = CallbackScope::new(context, proxy);
|
||||
if let Some(request) = request {
|
||||
scope.extension_scope = scope.extension_scope.with_request(request);
|
||||
}
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_extensions(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
extension_scope: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
scope.extension_scope = extension_scope;
|
||||
self.register_scope(context, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn bind_invocation(
|
||||
self: &Arc<Self>,
|
||||
call: &CallContext,
|
||||
origin: Option<InvocationContext>,
|
||||
extensions: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
invocation: Arc<dyn MiddlewareCallback>,
|
||||
) -> Result<MiddlewareBinding, gateway_core::engine::middleware::MiddlewareError> {
|
||||
let mut scope = CallbackScope::for_call(call);
|
||||
scope.extension_scope = extensions;
|
||||
scope.origin = origin;
|
||||
let scope = self
|
||||
.register_scope(call, scope)
|
||||
.map_err(|_| gateway_core::engine::middleware::MiddlewareError::Fault)?;
|
||||
self.bind_middleware(call.resource_scope_id.clone(), invocation, scope)
|
||||
}
|
||||
|
||||
fn register_scope(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
scope: CallbackScope,
|
||||
) -> Result<Arc<CallbackScope>, PluginFault> {
|
||||
let scope = Arc::new(scope);
|
||||
let mut scopes = self
|
||||
.scopes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
scopes.retain(|_, scope| scope.strong_count() != 0);
|
||||
if scopes
|
||||
.get(&context.resource_scope_id)
|
||||
.and_then(Weak::upgrade)
|
||||
.is_some()
|
||||
{
|
||||
return Err(denied());
|
||||
}
|
||||
scopes.insert(context.resource_scope_id.clone(), Arc::downgrade(&scope));
|
||||
Ok(scope)
|
||||
}
|
||||
}
|
||||
|
||||
impl CallbackHandler for PluginCallbacks {
|
||||
fn begin(&self, context: &CallContext, deadline: &CallDeadline) {
|
||||
let scope = self
|
||||
.scopes
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&context.resource_scope_id)
|
||||
.and_then(Weak::upgrade)
|
||||
.unwrap_or_else(|| {
|
||||
let mut scope = CallbackScope::for_call(context);
|
||||
if context.stage == gateway_plugin_sdk::Stage::Registration {
|
||||
scope.request_body_limit = self.registration_body_limit;
|
||||
}
|
||||
Arc::new(scope)
|
||||
});
|
||||
let middleware = self
|
||||
.pending_middleware
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&context.resource_scope_id);
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
context.call_id,
|
||||
Arc::new(CallResources::new(context, deadline, scope, middleware)),
|
||||
);
|
||||
}
|
||||
|
||||
fn finish(&self, context: &CallContext) {
|
||||
if let Some(call) = self
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&context.call_id)
|
||||
{
|
||||
call.close();
|
||||
}
|
||||
}
|
||||
|
||||
fn call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
method: String,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> BoxFuture<'static, Result<RpcReply, PluginFault>> {
|
||||
let call = self
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&context.call_id)
|
||||
.cloned();
|
||||
let http = self.http.clone();
|
||||
let network = self.network.clone();
|
||||
let maximum_payload = self.maximum_payload;
|
||||
let log = self.log.clone();
|
||||
let private_state = self.private_state.clone();
|
||||
let accounts = self.accounts.clone();
|
||||
let resources = self.resources.clone();
|
||||
let data = self.data.clone();
|
||||
let keys = self.keys.clone();
|
||||
let models = self.models.clone();
|
||||
let request_limits = self.request_limits.clone();
|
||||
let affinity = self.affinity.clone();
|
||||
let services = self.services.clone();
|
||||
let service_actor = self.service_actor.clone();
|
||||
Box::pin(async move {
|
||||
let call = call.ok_or_else(denied)?;
|
||||
let scope = &call.scope;
|
||||
let (middleware, is_private_state) = {
|
||||
let state = call
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.closed || call.deadline.is_elapsed() {
|
||||
return Err(denied());
|
||||
}
|
||||
if method == "host.log" {
|
||||
return log.record(&context, params, &payload);
|
||||
}
|
||||
let middleware = if method.starts_with("host.middleware.") {
|
||||
Some(state.middleware.clone().ok_or_else(denied)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let is_private_state = matches!(
|
||||
method.as_str(),
|
||||
"host.state.get" | "host.state.put" | "host.state.delete"
|
||||
);
|
||||
(middleware, is_private_state)
|
||||
};
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::middleware::http::DISPATCH_METHOD
|
||||
| gateway_plugin_sdk::call::middleware::http::BODY_READ_METHOD
|
||||
| gateway_plugin_sdk::call::middleware::http::BODY_CLOSE_METHOD
|
||||
| gateway_plugin_sdk::call::middleware::http::BODY_CREATE_METHOD
|
||||
| gateway_plugin_sdk::call::middleware::http::BODY_WRITE_METHOD
|
||||
) {
|
||||
return tokio::select! {
|
||||
biased;
|
||||
() = call.cancellation.cancelled() => Err(PluginFault::new(ErrorCode::Cancelled, "HTTP callback cancelled")),
|
||||
result = http_dispatch::call(&services, &context, &call, &method, params, payload, maximum_payload) => result,
|
||||
};
|
||||
}
|
||||
if let Some(middleware) = middleware {
|
||||
return middleware
|
||||
.invoke(method, params, payload, maximum_payload)
|
||||
.await;
|
||||
}
|
||||
if method == gateway_plugin_sdk::call::services::CALL_METHOD {
|
||||
return services
|
||||
.call(&context, &call, service_actor, params, &payload)
|
||||
.await;
|
||||
}
|
||||
if is_private_state {
|
||||
return private_state.call(&method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::resources::GROUP_ENSURE
|
||||
| gateway_plugin_sdk::call::resources::GROUP_MEMBERS
|
||||
| gateway_plugin_sdk::call::resources::KEY_ENSURE
|
||||
) {
|
||||
return resources.call(&context, &method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
"host.keys.list"
|
||||
| gateway_plugin_sdk::call::key_budgets::RESET
|
||||
| gateway_plugin_sdk::call::key_budgets::GET
|
||||
| gateway_plugin_sdk::call::key_budgets::UPDATE_LIMITS
|
||||
) {
|
||||
return keys.call(&context, &method, params, &payload).await;
|
||||
}
|
||||
if method == "host.models.list" {
|
||||
return models.call(&context, &call, &method, params, payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
"host.model.execute"
|
||||
| "host.model.execute_stream"
|
||||
| "host.model.stream_read"
|
||||
| "host.model.stream_close"
|
||||
) {
|
||||
return models.call(&context, &call, &method, params, payload).await;
|
||||
}
|
||||
if method == "host.affinity.lookup" {
|
||||
return affinity.call(&context, params, &payload).await;
|
||||
}
|
||||
let scope = Some(scope)
|
||||
.filter(|scope| scope.authorizes(&context))
|
||||
.ok_or_else(denied)?;
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
gateway_plugin_sdk::call::data::ACCOUNTS_LIST
|
||||
| gateway_plugin_sdk::call::data::KEYS_GET
|
||||
| gateway_plugin_sdk::call::data::QUOTA_GET
|
||||
| gateway_plugin_sdk::call::data::QUOTA_REFRESH
|
||||
) {
|
||||
return data.call(&method, params, &payload).await;
|
||||
}
|
||||
if matches!(
|
||||
method.as_str(),
|
||||
"host.auth.list" | "host.auth.get" | "host.auth.get_runtime" | "host.auth.save"
|
||||
) {
|
||||
return accounts
|
||||
.call(&context, scope, &method, params, &payload)
|
||||
.await;
|
||||
}
|
||||
let http = http::HttpCallbacks {
|
||||
client: &http,
|
||||
network: &network,
|
||||
scope,
|
||||
call: &call,
|
||||
maximum_payload,
|
||||
request_limits: &request_limits,
|
||||
};
|
||||
if method.starts_with("host.upstream.") {
|
||||
return upstream::dispatch(&http, &method, params, payload).await;
|
||||
}
|
||||
http.dispatch(&method, params, payload).await
|
||||
})
|
||||
}
|
||||
}
|
||||
pub(crate) use ports::{PluginCallbackPorts, PluginCallbacks};
|
||||
|
||||
fn denied() -> PluginFault {
|
||||
PluginFault::new(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 嵌套模型执行;只驱动 Core 会话,不拥有路由、重试或计费规则
|
||||
//! 模型目录与执行适配;身份解析交给客户端端口,路由与计费沿用 Core
|
||||
|
||||
use std::sync::{Arc, OnceLock, Weak};
|
||||
|
||||
@@ -9,11 +9,10 @@ use gateway_core::{
|
||||
engine::{
|
||||
CommitRequirement, ModelRequestId,
|
||||
execution::{
|
||||
BoundModelExecutionContext, ClientTransport, ExecutionRequestMetadata, ExecutionSession,
|
||||
AuthenticatedClient, ClientTransport, ExecutionRequestMetadata, ExecutionSession,
|
||||
},
|
||||
nested::{
|
||||
BoundModelExecutionBinding, BoundModelExecutionRequest, NestedModelExecutionPort,
|
||||
NestedModelExecutionRequest,
|
||||
BoundModelExecutionRequest, NestedModelExecutionPort, NestedModelExecutionRequest,
|
||||
},
|
||||
},
|
||||
event::ProviderEvent,
|
||||
@@ -22,13 +21,12 @@ use gateway_core::{
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, ProtocolPayload,
|
||||
RawJsonPayload, StandaloneSearchRequest,
|
||||
},
|
||||
policy::ClientApiKeyId,
|
||||
routing::PublicModelId,
|
||||
};
|
||||
use gateway_plugin_sdk::{
|
||||
CallContext, ErrorCode, PluginFault,
|
||||
call::host::{
|
||||
ModelEventBatch, ModelExecuteRequest, ModelExecuteResult, ModelListRequest,
|
||||
self as wire, ModelEventBatch, ModelExecuteRequest, ModelExecuteResult, ModelListRequest,
|
||||
ModelListResult, ModelOperation, ModelStreamCloseRequest, ModelStreamReadRequest,
|
||||
ModelStreamReadResult, ModelStreamResult,
|
||||
},
|
||||
@@ -63,6 +61,7 @@ impl PluginModelPortSlot {
|
||||
|
||||
pub(super) struct PluginModels {
|
||||
slot: Arc<PluginModelPortSlot>,
|
||||
clients: Arc<super::client::PluginClients>,
|
||||
request_limits: Arc<super::request_limits::RequestLimits>,
|
||||
maximum_payload: usize,
|
||||
}
|
||||
@@ -70,11 +69,13 @@ pub(super) struct PluginModels {
|
||||
impl PluginModels {
|
||||
pub(super) fn new(
|
||||
slot: Arc<PluginModelPortSlot>,
|
||||
clients: Arc<super::client::PluginClients>,
|
||||
request_limits: Arc<super::request_limits::RequestLimits>,
|
||||
maximum_payload: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
slot,
|
||||
clients,
|
||||
request_limits,
|
||||
maximum_payload,
|
||||
}
|
||||
@@ -89,7 +90,7 @@ impl PluginModels {
|
||||
payload: Vec<u8>,
|
||||
) -> Result<RpcReply, PluginFault> {
|
||||
match method {
|
||||
"host.models.list" => {
|
||||
wire::MODELS_LIST => {
|
||||
if !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -102,19 +103,14 @@ impl PluginModels {
|
||||
return Err(invalid());
|
||||
}
|
||||
let port = self.slot.upgrade()?;
|
||||
let bound = port
|
||||
.bind(BoundModelExecutionBinding {
|
||||
settings: call.request_settings(),
|
||||
client_key_id: ClientApiKeyId::new(request.client_key_id)
|
||||
.map_err(|_| invalid())?,
|
||||
initiating_plugin_instance_id: context.instance_id.clone(),
|
||||
cancellation: call.cancellation.child_token(),
|
||||
extension_scope: call.scope.extension_scope.clone(),
|
||||
})
|
||||
.await
|
||||
.map_err(super::error::gateway)?;
|
||||
let client = self.clients.identity(call, request.client_key_id).await?;
|
||||
let models = port
|
||||
.models(bound, request.protocol, request.client_version)
|
||||
.models(
|
||||
client,
|
||||
call.cancellation.clone(),
|
||||
request.protocol,
|
||||
request.client_version,
|
||||
)
|
||||
.await
|
||||
.map_err(super::error::gateway)?;
|
||||
encode_result(ModelListResult {
|
||||
@@ -124,7 +120,7 @@ impl PluginModels {
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
"host.model.execute" | "host.model.execute_stream" => {
|
||||
wire::MODEL_EXECUTE | wire::MODEL_EXECUTE_STREAM => {
|
||||
let mut request: ModelExecuteRequest =
|
||||
serde_json::from_value(params).map_err(|_| invalid())?;
|
||||
if payload.is_empty() {
|
||||
@@ -139,11 +135,14 @@ impl PluginModels {
|
||||
}
|
||||
};
|
||||
let port = self.slot.upgrade()?;
|
||||
let stream = method == "host.model.execute_stream";
|
||||
let stream = method == wire::MODEL_EXECUTE_STREAM;
|
||||
let started = if let Some(client_key_id) = request.client_key_id.take() {
|
||||
let bound = self.binding(&port, context, call, client_key_id).await?;
|
||||
check_size(bound.request_body_limit())?;
|
||||
port.start_bound(self.bound_request(request, payload, stream, bound)?)
|
||||
let client = self.clients.identity(call, client_key_id).await?;
|
||||
check_size(client.snapshot().responses_max_decompressed_body_bytes())?;
|
||||
let request = self.request_parts(request, payload, stream)?;
|
||||
let request = Self::bound_request(context, call, request, client);
|
||||
call.observe_dispatch()?;
|
||||
port.start_bound(request)
|
||||
.await
|
||||
.map_err(super::error::gateway)?
|
||||
} else {
|
||||
@@ -151,7 +150,7 @@ impl PluginModels {
|
||||
let request = self.request(context, request, payload, stream)?;
|
||||
port.start(request).await.map_err(super::error::gateway)?
|
||||
};
|
||||
if method == "host.model.execute" {
|
||||
if method == wire::MODEL_EXECUTE {
|
||||
execute_buffered(started, self.maximum_payload).await
|
||||
} else {
|
||||
let stream = Arc::new(ModelStream::new(started.session, self.maximum_payload));
|
||||
@@ -175,7 +174,7 @@ impl PluginModels {
|
||||
})
|
||||
}
|
||||
}
|
||||
"host.model.stream_read" => {
|
||||
wire::MODEL_STREAM_READ => {
|
||||
if !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -214,7 +213,7 @@ impl PluginModels {
|
||||
payload: read.payload,
|
||||
})
|
||||
}
|
||||
"host.model.stream_close" => {
|
||||
wire::MODEL_STREAM_CLOSE => {
|
||||
if !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -237,44 +236,6 @@ impl PluginModels {
|
||||
}
|
||||
}
|
||||
|
||||
async fn binding(
|
||||
&self,
|
||||
port: &Arc<dyn NestedModelExecutionPort>,
|
||||
context: &CallContext,
|
||||
call: &CallResources,
|
||||
client_key_id: String,
|
||||
) -> Result<BoundModelExecutionContext, PluginFault> {
|
||||
let key = ClientApiKeyId::new(client_key_id.clone()).map_err(|_| invalid())?;
|
||||
let settings = call.request_settings();
|
||||
let mut bindings = call.model_bindings.lock().await;
|
||||
if let Some(bound) = bindings
|
||||
.get(&client_key_id)
|
||||
.filter(|bound| bound.request_settings() == settings.as_ref())
|
||||
{
|
||||
return Ok(bound.clone());
|
||||
}
|
||||
if !bindings.contains_key(&client_key_id) && bindings.len() >= MAX_MODEL_STREAMS_PER_CALL {
|
||||
return Err(PluginFault::new(
|
||||
ErrorCode::Capacity,
|
||||
"model identity capacity is exhausted",
|
||||
));
|
||||
}
|
||||
let bound = port
|
||||
.bind(BoundModelExecutionBinding {
|
||||
settings,
|
||||
client_key_id: key,
|
||||
initiating_plugin_instance_id: context.instance_id.clone(),
|
||||
cancellation: call.cancellation.child_token(),
|
||||
extension_scope: call.scope.extension_scope.clone(),
|
||||
})
|
||||
.await
|
||||
.map_err(super::error::gateway)?;
|
||||
// 调用持有身份直到 RPC 结束,流式执行不会因单次 callback 返回而提前取消;
|
||||
// 同一 Key 复用身份快照,每次执行由 Core 创建独立调用图
|
||||
bindings.insert(client_key_id, bound.clone());
|
||||
Ok(bound)
|
||||
}
|
||||
|
||||
fn request(
|
||||
&self,
|
||||
context: &CallContext,
|
||||
@@ -310,21 +271,22 @@ impl PluginModels {
|
||||
}
|
||||
|
||||
fn bound_request(
|
||||
&self,
|
||||
request: ModelExecuteRequest,
|
||||
body: Vec<u8>,
|
||||
stream: bool,
|
||||
context: BoundModelExecutionContext,
|
||||
) -> Result<BoundModelExecutionRequest, PluginFault> {
|
||||
let request = self.request_parts(request, body, stream)?;
|
||||
Ok(BoundModelExecutionRequest {
|
||||
context,
|
||||
context: &CallContext,
|
||||
call: &CallResources,
|
||||
request: ModelRequestParts,
|
||||
client: AuthenticatedClient,
|
||||
) -> BoundModelExecutionRequest {
|
||||
BoundModelExecutionRequest {
|
||||
client,
|
||||
initiating_plugin_instance_id: context.instance_id.clone(),
|
||||
cancellation: call.cancellation.clone(),
|
||||
extension_scope: call.scope.extension_scope.clone(),
|
||||
public_model: request.public_model,
|
||||
operation: request.operation,
|
||||
metadata: request.metadata,
|
||||
provider: request.provider,
|
||||
account: request.account,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn request_parts(
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//! 组装实例宿主端口及调用资源索引,保持资源与插件实例同寿命
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex, Weak},
|
||||
};
|
||||
|
||||
use gateway_admin::model::AdminError;
|
||||
use gateway_host::outbound::{HttpClient, NetworkPolicy};
|
||||
|
||||
use super::{
|
||||
CallResources, CallbackScope, MiddlewareCallback, PluginAccountPortSlot,
|
||||
PluginAffinityPortSlot, PluginClientIdentityPortSlot, PluginClientKeyPortSlot,
|
||||
PluginModelPortSlot, PluginResourcePorts,
|
||||
};
|
||||
use super::{
|
||||
accounts, affinity, client, data, keys, log, model, private_state, request_limits, resources,
|
||||
services,
|
||||
};
|
||||
|
||||
pub(crate) struct PluginCallbackPorts {
|
||||
pub(crate) services: Arc<services::ServicePorts>,
|
||||
pub(crate) http: Arc<HttpClient>,
|
||||
pub(crate) accounts: Arc<PluginAccountPortSlot>,
|
||||
pub(crate) keys: Arc<PluginClientKeyPortSlot>,
|
||||
pub(crate) clients: Arc<PluginClientIdentityPortSlot>,
|
||||
pub(crate) models: Arc<PluginModelPortSlot>,
|
||||
pub(crate) affinity: Arc<PluginAffinityPortSlot>,
|
||||
pub(crate) resources: Arc<PluginResourcePorts>,
|
||||
}
|
||||
|
||||
pub(crate) struct PluginCallbacks {
|
||||
pub(crate) registration_body_limit: Option<usize>,
|
||||
pub(super) services: Arc<services::ServicePorts>,
|
||||
pub(super) service_actor: gateway_admin::model::usage_data::UsageActor,
|
||||
pub(super) accounts: Arc<accounts::PluginAccounts>,
|
||||
pub(super) resources: Arc<resources::PluginResources>,
|
||||
pub(super) data: Arc<data::PluginData>,
|
||||
pub(super) keys: Arc<keys::PluginClientKeys>,
|
||||
pub(super) affinity: Arc<affinity::PluginAffinity>,
|
||||
pub(super) log: Arc<log::PluginLog>,
|
||||
pub(super) clients: Arc<client::PluginClients>,
|
||||
pub(super) models: Arc<model::PluginModels>,
|
||||
pub(super) private_state: Arc<private_state::PluginPrivateState>,
|
||||
pub(super) http: Arc<HttpClient>,
|
||||
pub(super) network: Arc<NetworkPolicy>,
|
||||
pub(super) scopes: Mutex<BTreeMap<String, Weak<CallbackScope>>>,
|
||||
pub(super) pending_middleware: Mutex<BTreeMap<String, Arc<dyn MiddlewareCallback>>>,
|
||||
pub(super) calls: Mutex<BTreeMap<u64, Arc<CallResources>>>,
|
||||
pub(super) maximum_payload: usize,
|
||||
pub(super) request_limits: Arc<request_limits::RequestLimits>,
|
||||
}
|
||||
|
||||
impl PluginCallbacks {
|
||||
pub(crate) fn new(
|
||||
instance: &gateway_admin::model::plugins::instances::PluginInstance,
|
||||
maximum_payload: usize,
|
||||
manifest: &gateway_plugin_sdk::Manifest,
|
||||
log_slots: Arc<tokio::sync::Semaphore>,
|
||||
private_state: Arc<private_state::PluginPrivateState>,
|
||||
ports: PluginCallbackPorts,
|
||||
request_limits: Arc<request_limits::RequestLimits>,
|
||||
) -> Result<Self, AdminError> {
|
||||
let clients = Arc::new(client::PluginClients::new(ports.clients));
|
||||
Ok(Self {
|
||||
registration_body_limit: None,
|
||||
services: ports.services,
|
||||
service_actor: gateway_admin::model::usage_data::UsageActor::Plugin {
|
||||
publisher: manifest.publisher.clone(),
|
||||
name: manifest.name.clone(),
|
||||
instance_id: instance.id.clone(),
|
||||
},
|
||||
resources: Arc::new(resources::PluginResources::new(instance, ports.resources)),
|
||||
data: Arc::new(data::PluginData::new(
|
||||
ports.accounts.clone(),
|
||||
ports.keys.clone(),
|
||||
)),
|
||||
accounts: Arc::new(accounts::PluginAccounts::new(ports.accounts)),
|
||||
keys: Arc::new(keys::PluginClientKeys::new(instance, ports.keys)),
|
||||
affinity: Arc::new(affinity::PluginAffinity::new(ports.affinity)),
|
||||
log: Arc::new(
|
||||
log::PluginLog::new(manifest, log_slots)
|
||||
.map_err(|_| AdminError::invalid("插件清单身份不合法"))?,
|
||||
),
|
||||
clients: clients.clone(),
|
||||
models: Arc::new(model::PluginModels::new(
|
||||
ports.models,
|
||||
clients,
|
||||
request_limits.clone(),
|
||||
maximum_payload,
|
||||
)),
|
||||
private_state,
|
||||
http: ports.http,
|
||||
network: Arc::new(NetworkPolicy::unrestricted()),
|
||||
scopes: Mutex::new(BTreeMap::new()),
|
||||
pending_middleware: Mutex::new(BTreeMap::new()),
|
||||
calls: Mutex::new(BTreeMap::new()),
|
||||
maximum_payload,
|
||||
request_limits,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,8 @@ use gateway_admin::{
|
||||
use gateway_plugin_sdk::{
|
||||
ErrorCode, Manifest, PluginFault,
|
||||
call::host::{
|
||||
StateDeleteRequest, StateDeleteResult, StateGetRequest, StateGetResult, StatePutRequest,
|
||||
StatePutResult, StateRecord,
|
||||
self as wire, StateDeleteRequest, StateDeleteResult, StateGetRequest, StateGetResult,
|
||||
StatePutRequest, StatePutResult, StateRecord,
|
||||
},
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
@@ -99,7 +99,7 @@ impl PluginPrivateState {
|
||||
.clone()
|
||||
.ok_or_else(denied_fault)?;
|
||||
let result = match method {
|
||||
"host.state.get" => {
|
||||
wire::STATE_GET => {
|
||||
let request: StateGetRequest =
|
||||
serde_json::from_value(params).map_err(|_| invalid_fault())?;
|
||||
self.authorize(&request.namespace)?;
|
||||
@@ -115,7 +115,7 @@ impl PluginPrivateState {
|
||||
});
|
||||
serde_json::to_value(StateGetResult { record }).map_err(|_| internal_fault())?
|
||||
}
|
||||
"host.state.put" => {
|
||||
wire::STATE_PUT => {
|
||||
let request: StatePutRequest =
|
||||
serde_json::from_value(params).map_err(|_| invalid_fault())?;
|
||||
self.authorize(&request.namespace)?;
|
||||
@@ -141,7 +141,7 @@ impl PluginPrivateState {
|
||||
})
|
||||
.map_err(|_| internal_fault())?
|
||||
}
|
||||
"host.state.delete" => {
|
||||
wire::STATE_DELETE => {
|
||||
let request: StateDeleteRequest =
|
||||
serde_json::from_value(params).map_err(|_| invalid_fault())?;
|
||||
self.authorize(&request.namespace)?;
|
||||
|
||||
@@ -7,14 +7,7 @@ use gateway_core::{
|
||||
};
|
||||
use gateway_plugin_sdk::{CallContext, Stage};
|
||||
|
||||
/// 所有中间件共享的父调用身份;不混入协议操作或服务参数
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct InvocationContext {
|
||||
pub request_id: String,
|
||||
pub call_id: String,
|
||||
pub cancellation: gateway_core::lifecycle::CancellationToken,
|
||||
pub plan: gateway_core::engine::middleware::FrozenMiddlewarePlan,
|
||||
}
|
||||
pub(crate) use gateway_core::engine::extensions::InvocationOrigin as InvocationContext;
|
||||
|
||||
/// 一次回调的调用链、账号与网络事实,由操作持有到完成
|
||||
pub(crate) struct CallbackScope {
|
||||
@@ -25,7 +18,8 @@ pub(crate) struct CallbackScope {
|
||||
credential_revision: Option<u64>,
|
||||
pub(super) proxy: Option<OutboundProxy>,
|
||||
pub(super) extension_scope: gateway_core::engine::extensions::ExtensionCallScope,
|
||||
execution_effects: Option<Arc<ExecutionEffects>>,
|
||||
pub(super) execution_effects: Option<Arc<ExecutionEffects>>,
|
||||
pub(super) settings: Option<gateway_core::routing::request_settings::RequestSettings>,
|
||||
pub(super) upstream: Option<Arc<super::upstream::ManagedUpstream>>,
|
||||
}
|
||||
|
||||
@@ -37,14 +31,13 @@ impl CallbackScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn child_extensions(
|
||||
pub(super) fn check_child_depth(
|
||||
&self,
|
||||
instance_id: &str,
|
||||
) -> Result<gateway_core::engine::extensions::ExtensionCallScope, gateway_plugin_sdk::PluginFault>
|
||||
{
|
||||
) -> Result<(), gateway_plugin_sdk::PluginFault> {
|
||||
// 上游适配器在进入回调前已登记当前实例;主动子调用沿用该集合
|
||||
if self.extension_scope.contains(instance_id) {
|
||||
return Ok(self.extension_scope.clone());
|
||||
return Ok(());
|
||||
}
|
||||
if self.extension_scope.len()
|
||||
>= gateway_core::engine::extensions::ExtensionCallScope::MAXIMUM_DEPTH
|
||||
@@ -54,9 +47,7 @@ impl CallbackScope {
|
||||
"child call depth exceeded",
|
||||
));
|
||||
}
|
||||
self.extension_scope
|
||||
.extending(instance_id.to_owned())
|
||||
.ok_or_else(super::invalid)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn new(context: &CallContext, proxy: Option<OutboundProxy>) -> Self {
|
||||
@@ -69,6 +60,7 @@ impl CallbackScope {
|
||||
proxy,
|
||||
extension_scope: Default::default(),
|
||||
execution_effects: None,
|
||||
settings: None,
|
||||
upstream: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,27 +63,18 @@ impl ServicePorts {
|
||||
)
|
||||
})?;
|
||||
let call = registry.call(&request.operation, request.input);
|
||||
let scope = &resources.scope;
|
||||
let origin = scope.origin.as_ref();
|
||||
resources.scope.check_child_depth(&context.instance_id)?;
|
||||
let origin = resources.child_origin(context)?;
|
||||
let parent = gateway_admin::public_service::Origin {
|
||||
request_id: origin.map_or_else(
|
||||
|| {
|
||||
context
|
||||
.request_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| context.resource_scope_id.clone())
|
||||
},
|
||||
|origin| origin.request_id.clone(),
|
||||
),
|
||||
call_id: origin.map_or_else(
|
||||
|| format!("{}:{}", context.resource_scope_id, context.call_id),
|
||||
|origin| origin.call_id.clone(),
|
||||
),
|
||||
cancellation: resources.cancellation.clone(),
|
||||
extensions: scope.child_extensions(&context.instance_id)?,
|
||||
plan: origin.map_or(gateway_admin::public_service::Plan::Current, |origin| {
|
||||
gateway_admin::public_service::Plan::Frozen(Some(origin.plan.clone()))
|
||||
}),
|
||||
request_id: origin.request_id,
|
||||
call_id: origin.call_id,
|
||||
cancellation: origin.cancellation,
|
||||
extensions: origin.extensions,
|
||||
plan: origin
|
||||
.plan
|
||||
.map_or(gateway_admin::public_service::Plan::Current, |plan| {
|
||||
gateway_admin::public_service::Plan::Frozen(Some(plan))
|
||||
}),
|
||||
};
|
||||
// 即使终端只读,服务中间件也可提交写入;不存在的操作不会进入调用链
|
||||
if registry
|
||||
|
||||
@@ -193,26 +193,31 @@ pub(super) async fn dispatch(
|
||||
.map(|remaining| remaining.min(Duration::from_secs(120)))
|
||||
};
|
||||
match method {
|
||||
"host.upstream.http.do" | "host.upstream.http.do_stream" => {
|
||||
adapter::HTTP_DO | adapter::HTTP_DO_STREAM => {
|
||||
let request = serde_json::from_value(params).map_err(|_| invalid())?;
|
||||
let maximum_body_bytes = http.body_limit(&payload)?;
|
||||
let (request, purpose) = managed.request(request, payload, maximum_body_bytes)?;
|
||||
http.send(request, method.ends_with("do_stream"), Some(purpose))
|
||||
.await
|
||||
http.send(
|
||||
request,
|
||||
method == adapter::HTTP_DO_STREAM,
|
||||
Some(purpose),
|
||||
scope.proxy.as_ref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
"host.upstream.http.stream_read" | "host.upstream.http.stream_close" => {
|
||||
adapter::HTTP_STREAM_READ | adapter::HTTP_STREAM_CLOSE => {
|
||||
http.dispatch(
|
||||
if method.ends_with("stream_read") {
|
||||
"host.http.stream_read"
|
||||
if method == adapter::HTTP_STREAM_READ {
|
||||
wire::HTTP_STREAM_READ
|
||||
} else {
|
||||
"host.http.stream_close"
|
||||
wire::HTTP_STREAM_CLOSE
|
||||
},
|
||||
params,
|
||||
payload,
|
||||
)
|
||||
.await
|
||||
}
|
||||
"host.upstream.websocket.open" => {
|
||||
adapter::WEBSOCKET_OPEN => {
|
||||
if !payload.is_empty() || !managed.websocket_allowed {
|
||||
return Err(denied());
|
||||
}
|
||||
@@ -298,7 +303,7 @@ pub(super) async fn dispatch(
|
||||
payload,
|
||||
})
|
||||
}
|
||||
"host.upstream.websocket.send" => {
|
||||
adapter::WEBSOCKET_SEND => {
|
||||
let message: adapter::UpstreamWebSocketMessage =
|
||||
serde_json::from_value(params).map_err(|_| invalid())?;
|
||||
let message = match message.kind {
|
||||
@@ -322,7 +327,7 @@ pub(super) async fn dispatch(
|
||||
payload: vec![],
|
||||
})
|
||||
}
|
||||
"host.upstream.websocket.read" => {
|
||||
adapter::WEBSOCKET_READ => {
|
||||
if params != serde_json::json!({}) || !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -353,7 +358,7 @@ pub(super) async fn dispatch(
|
||||
payload,
|
||||
})
|
||||
}
|
||||
"host.upstream.websocket.close" => {
|
||||
adapter::WEBSOCKET_CLOSE => {
|
||||
if params != serde_json::json!({}) || !payload.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ use gateway_admin::model::{
|
||||
AdminError, AdminErrorKind, Revision,
|
||||
plugins::instances::{
|
||||
PluginInstance, PluginInstanceRuntime, PluginInstanceRuntimeFailure,
|
||||
PluginInstanceRuntimeStatus, PluginInstanceSnapshot,
|
||||
PluginInstanceRuntimeLoad, PluginInstanceRuntimeStatus, PluginInstanceSnapshot,
|
||||
},
|
||||
};
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
@@ -227,6 +227,17 @@ impl InstanceRuntimeProjection<'_> {
|
||||
actual_artifact_sha256: active.map(|instance| instance.artifact_sha256.clone()),
|
||||
failure,
|
||||
draining_revisions: draining_revisions.into_iter().collect(),
|
||||
load: active.map(|instance| {
|
||||
let load = instance.session.load();
|
||||
PluginInstanceRuntimeLoad {
|
||||
active_calls: load.active_calls,
|
||||
executing_calls: load.executing_calls,
|
||||
queued_calls: load.queued_calls,
|
||||
resuming_calls: load.resuming_calls,
|
||||
pending_callbacks: load.pending_callbacks,
|
||||
active_streams: load.active_streams,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,18 +16,22 @@ use gateway_admin::{
|
||||
use gateway_plugin_sdk::{Handshake, Stage, call::registration::Registration};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
const PROCESS_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct StartedInstance {
|
||||
instance: PluginInstance,
|
||||
package: Arc<crate::PreparedPackage>,
|
||||
callbacks: Arc<PluginCallbacks>,
|
||||
prepared: PreparedInstance,
|
||||
}
|
||||
|
||||
impl PluginRuntime {
|
||||
pub(super) async fn prepare_instance(
|
||||
async fn start_instance(
|
||||
&self,
|
||||
instance: PluginInstance,
|
||||
target_revision: Revision,
|
||||
request_body_limit: Option<usize>,
|
||||
) -> Result<Arc<PreparedInstance>, AdminError> {
|
||||
let mut observer_entries = Vec::new();
|
||||
let mut commands = Vec::new();
|
||||
let mut management = Vec::new();
|
||||
let mut policy_entries = Vec::new();
|
||||
let mut authentication_entries = Vec::new();
|
||||
) -> Result<StartedInstance, AdminError> {
|
||||
let authorization_binding = instance_fingerprint(&instance)?;
|
||||
if self.restart_circuit_is_open(&RestartIdentity::new(
|
||||
instance.id.clone(),
|
||||
@@ -109,7 +113,6 @@ impl PluginRuntime {
|
||||
crate::compatibility::report(&manifest, &instance_id)?;
|
||||
let restart_identity =
|
||||
RestartIdentity::new(instance_id.clone(), authorization_binding.clone());
|
||||
let bindings = instance.bindings.clone();
|
||||
let owner = self
|
||||
.state
|
||||
.load_owner(
|
||||
@@ -158,15 +161,16 @@ impl PluginRuntime {
|
||||
&manifest,
|
||||
self.log_slots.clone(),
|
||||
private_state.clone(),
|
||||
PluginCallbackPorts::new(
|
||||
self.service_ports.clone(),
|
||||
self.http.clone(),
|
||||
self.account_ports.clone(),
|
||||
self.client_key_ports.clone(),
|
||||
self.model_ports.clone(),
|
||||
self.affinity_ports.clone(),
|
||||
self.resource_ports.clone(),
|
||||
),
|
||||
PluginCallbackPorts {
|
||||
services: self.service_ports.clone(),
|
||||
http: self.http.clone(),
|
||||
accounts: self.account_ports.clone(),
|
||||
keys: self.client_key_ports.clone(),
|
||||
clients: self.client_identity_port.clone(),
|
||||
models: self.model_ports.clone(),
|
||||
affinity: self.affinity_ports.clone(),
|
||||
resources: self.resource_ports.clone(),
|
||||
},
|
||||
self.request_limits.clone(),
|
||||
)?;
|
||||
callbacks.registration_body_limit = request_body_limit;
|
||||
@@ -181,23 +185,69 @@ impl PluginRuntime {
|
||||
lifecycle,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| AdminError::unavailable("插件进程握手失败"))?,
|
||||
.map_err(|error| match error {
|
||||
crate::RpcError::Budget(gateway_plugin_sdk::CallBudgetError::Invalid {
|
||||
stage,
|
||||
milliseconds,
|
||||
}) => AdminError::invalid(format!(
|
||||
"插件阶段 {stage:?} 的调用预算无效:{milliseconds} 毫秒"
|
||||
)),
|
||||
_ => AdminError::unavailable("插件进程握手失败"),
|
||||
})?,
|
||||
);
|
||||
let mut prepared = PreparedInstance {
|
||||
let prepared = PreparedInstance {
|
||||
instance_id: instance_id.clone(),
|
||||
artifact_sha256: instance.artifact_sha256.clone(),
|
||||
revision: instance.revision,
|
||||
session: session.clone(),
|
||||
session,
|
||||
private_state: private_state.clone(),
|
||||
maintenance: manifest
|
||||
.contributes
|
||||
.contains_key(&gateway_plugin_sdk::Capability::Maintenance),
|
||||
contributions: PreparedContributions::default(),
|
||||
};
|
||||
Ok(StartedInstance {
|
||||
instance,
|
||||
package,
|
||||
callbacks,
|
||||
prepared,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_instance(
|
||||
&self,
|
||||
instance: PluginInstance,
|
||||
target_revision: Revision,
|
||||
request_body_limit: Option<usize>,
|
||||
) -> Result<Arc<PreparedInstance>, AdminError> {
|
||||
// 启动控制期限只包住制品准备与握手,Ready 后每次注册使用协商预算
|
||||
let StartedInstance {
|
||||
instance,
|
||||
package,
|
||||
callbacks,
|
||||
mut prepared,
|
||||
} = tokio::time::timeout(
|
||||
PROCESS_PREPARATION_TIMEOUT,
|
||||
self.start_instance(instance, target_revision, request_body_limit),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| AdminError::unavailable("插件启动超时"))??;
|
||||
let session = &prepared.session;
|
||||
let manifest = package.package().manifest();
|
||||
let plugin_id = manifest
|
||||
.plugin_id()
|
||||
.map_err(|_| AdminError::invalid("插件身份无效"))?;
|
||||
let instance_id = &instance.id;
|
||||
let bindings = &instance.bindings;
|
||||
let mut observer_entries = Vec::new();
|
||||
let mut commands = Vec::new();
|
||||
let mut management = Vec::new();
|
||||
let mut policy_entries = Vec::new();
|
||||
let mut authentication_entries = Vec::new();
|
||||
let registration = session
|
||||
.call(
|
||||
"plugin.register",
|
||||
session.context(Stage::Registration, Duration::from_secs(5)),
|
||||
session.context(Stage::Registration, session.budget(Stage::Registration)),
|
||||
serde_json::json!({}),
|
||||
vec![],
|
||||
)
|
||||
@@ -211,7 +261,7 @@ impl PluginRuntime {
|
||||
commands.extend(
|
||||
crate::adapter::command_line::prepare(
|
||||
&instance,
|
||||
&manifest,
|
||||
manifest,
|
||||
session.clone(),
|
||||
callbacks.clone(),
|
||||
)
|
||||
@@ -228,8 +278,8 @@ impl PluginRuntime {
|
||||
management.push(entry);
|
||||
}
|
||||
if let Some(entry) = crate::adapter::frontend_authentication::prepare_entry(
|
||||
&manifest,
|
||||
&bindings,
|
||||
manifest,
|
||||
bindings,
|
||||
session.clone(),
|
||||
callbacks.clone(),
|
||||
)
|
||||
@@ -238,35 +288,34 @@ impl PluginRuntime {
|
||||
authentication_entries.push(entry);
|
||||
}
|
||||
if let Some(entry) = crate::adapter::observer::compile_entry(
|
||||
&manifest,
|
||||
manifest,
|
||||
&plugin_id,
|
||||
&instance_id,
|
||||
&bindings,
|
||||
instance_id,
|
||||
bindings,
|
||||
session.clone(),
|
||||
callbacks.clone(),
|
||||
)? {
|
||||
observer_entries.push(entry);
|
||||
}
|
||||
policy_entries.extend(crate::adapter::policy::compile_entries(
|
||||
&manifest,
|
||||
&instance_id,
|
||||
&bindings,
|
||||
manifest,
|
||||
instance_id,
|
||||
bindings,
|
||||
session.clone(),
|
||||
callbacks.clone(),
|
||||
)?);
|
||||
let middleware_entries = crate::adapter::middleware::compile_entries(
|
||||
&manifest,
|
||||
&instance_id,
|
||||
&bindings,
|
||||
manifest,
|
||||
instance_id,
|
||||
bindings,
|
||||
session.clone(),
|
||||
callbacks.clone(),
|
||||
)?;
|
||||
let model_aliases =
|
||||
crate::adapter::catalog::prepare(&manifest, &instance, &session).await?;
|
||||
let model_aliases = crate::adapter::catalog::prepare(manifest, &instance, session).await?;
|
||||
let upstream_entries = crate::adapter::upstream_adapter::prepare(
|
||||
&manifest,
|
||||
manifest,
|
||||
&instance,
|
||||
Arc::clone(&session),
|
||||
Arc::clone(session),
|
||||
Arc::clone(&callbacks),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 对账只从已发布集合开始;通知合并,实例串行,失败和停用不阻塞其他实例
|
||||
|
||||
use super::{PluginRuntime, set::PreparedSet};
|
||||
use super::{
|
||||
PluginRuntime,
|
||||
set::{PreparedInstance, PreparedSet},
|
||||
};
|
||||
use crate::RpcSession;
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use futures::{StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
runtime::RuntimeSnapshotHandle,
|
||||
@@ -13,12 +16,51 @@ use gateway_core::{
|
||||
},
|
||||
};
|
||||
use gateway_plugin_sdk::Stage;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::{sync::Notify, task::JoinSet};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Duration};
|
||||
use tokio::{sync::Notify, task::JoinHandle};
|
||||
|
||||
const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const RETRY_DELAY: Duration = Duration::from_secs(5);
|
||||
const CALL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct InstanceTask {
|
||||
instance: Arc<PreparedInstance>,
|
||||
dirty: Arc<Notify>,
|
||||
cancellation: CancellationToken,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl InstanceTask {
|
||||
fn start(instance: Arc<PreparedInstance>) -> Self {
|
||||
let dirty = Arc::new(Notify::new());
|
||||
dirty.notify_one();
|
||||
let cancellation = CancellationToken::new();
|
||||
let task = tokio::spawn(reconcile(
|
||||
Arc::clone(&instance.session),
|
||||
Arc::clone(&dirty),
|
||||
cancellation.clone(),
|
||||
));
|
||||
Self {
|
||||
instance,
|
||||
dirty,
|
||||
cancellation,
|
||||
task,
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(mut self) {
|
||||
self.cancellation.cancel();
|
||||
let _ = (&mut self.task).await;
|
||||
// 宿主资源在取消时同步撤销,替换任务还须等对端确认或进程退出
|
||||
self.instance.session.settle_cancellations().await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InstanceTask {
|
||||
fn drop(&mut self) {
|
||||
self.cancellation.cancel();
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
struct MaintenanceTask {
|
||||
runtime: Arc<PluginRuntime>,
|
||||
@@ -59,18 +101,12 @@ impl DaemonTask for MaintenanceTask {
|
||||
let mut publications = self.snapshots.subscribe_publications();
|
||||
let mut active: Option<Arc<PreparedSet>> = None;
|
||||
let mut revision = None;
|
||||
let mut stop = CancellationToken::new();
|
||||
let mut tasks = JoinSet::new();
|
||||
let mut notifications = Vec::new();
|
||||
let mut tasks = BTreeMap::<String, InstanceTask>::new();
|
||||
loop {
|
||||
let snapshot = self.snapshots.acquire().ok();
|
||||
let reference = snapshot.as_ref().and_then(|s| s.extensions());
|
||||
let same_set = active.as_ref().map(|s| &s.id) == reference.map(|s| s.id());
|
||||
if !same_set {
|
||||
stop.cancel();
|
||||
while tasks.join_next().await.is_some() {}
|
||||
notifications.clear();
|
||||
stop = CancellationToken::new();
|
||||
active = match reference {
|
||||
Some(reference) => {
|
||||
Some(self.runtime.prepared_set(reference).await.map_err(|_| {
|
||||
@@ -79,17 +115,37 @@ impl DaemonTask for MaintenanceTask {
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
if let Some(set) = &active {
|
||||
for instance in set.sessions.iter().filter(|i| i.maintenance) {
|
||||
let dirty = Arc::new(Notify::new());
|
||||
dirty.notify_one();
|
||||
notifications.push(dirty.clone());
|
||||
tasks.spawn(reconcile(instance.session.clone(), dirty, stop.clone()));
|
||||
}
|
||||
let desired = active
|
||||
.iter()
|
||||
.flat_map(|set| &set.sessions)
|
||||
.filter(|instance| instance.maintenance)
|
||||
.map(|instance| (instance.instance_id.clone(), Arc::clone(instance)))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let retired = tasks
|
||||
.extract_if(.., |id, task| {
|
||||
// 发布层只复用完全相同且就绪的实例,Arc 身份也包含会话 incarnation
|
||||
!desired
|
||||
.get(id)
|
||||
.is_some_and(|instance| Arc::ptr_eq(instance, &task.instance))
|
||||
})
|
||||
.map(|(_, task)| task)
|
||||
.collect::<Vec<_>>();
|
||||
for task in &retired {
|
||||
task.cancellation.cancel();
|
||||
}
|
||||
} else if revision != snapshot.as_ref().map(|s| s.revision()) {
|
||||
for dirty in ¬ifications {
|
||||
dirty.notify_one();
|
||||
futures::future::join_all(retired.into_iter().map(InstanceTask::stop)).await;
|
||||
for task in tasks.values() {
|
||||
task.dirty.notify_one();
|
||||
}
|
||||
for (id, instance) in desired {
|
||||
tasks
|
||||
.entry(id)
|
||||
.or_insert_with(|| InstanceTask::start(instance));
|
||||
}
|
||||
}
|
||||
if same_set && revision != snapshot.as_ref().map(|s| s.revision()) {
|
||||
for task in tasks.values() {
|
||||
task.dirty.notify_one();
|
||||
}
|
||||
}
|
||||
revision = snapshot.as_ref().map(|s| s.revision());
|
||||
@@ -101,22 +157,39 @@ impl DaemonTask for MaintenanceTask {
|
||||
biased;
|
||||
() = cancellation.cancelled() => break,
|
||||
changed = publications.next() => if changed.is_none() { break; },
|
||||
result = tasks.join_next(), if !tasks.is_empty() => {
|
||||
if result.is_some() {
|
||||
stop.cancel();
|
||||
while tasks.join_next().await.is_some() {}
|
||||
result = next_finished(&mut tasks), if !tasks.is_empty() => {
|
||||
if let Some(instance_id) = result {
|
||||
tasks.remove(&instance_id);
|
||||
stop_all(tasks).await;
|
||||
return Err(WorkerTaskError::safe("plugin maintenance task stopped unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stop.cancel();
|
||||
while tasks.join_next().await.is_some() {}
|
||||
stop_all(tasks).await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_finished(tasks: &mut BTreeMap<String, InstanceTask>) -> Option<String> {
|
||||
let mut pending = tasks
|
||||
.iter_mut()
|
||||
.map(|(id, task)| async move {
|
||||
let _ = (&mut task.task).await;
|
||||
id.clone()
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
pending.next().await
|
||||
}
|
||||
|
||||
async fn stop_all(tasks: BTreeMap<String, InstanceTask>) {
|
||||
for task in tasks.values() {
|
||||
task.cancellation.cancel();
|
||||
}
|
||||
futures::future::join_all(tasks.into_values().map(InstanceTask::stop)).await;
|
||||
}
|
||||
|
||||
async fn reconcile(session: Arc<RpcSession>, dirty: Arc<Notify>, cancellation: CancellationToken) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -125,21 +198,38 @@ async fn reconcile(session: Arc<RpcSession>, dirty: Arc<Notify>, cancellation: C
|
||||
() = dirty.notified() => {},
|
||||
() = tokio::time::sleep(RECONCILE_INTERVAL) => {},
|
||||
}
|
||||
let context = session.context(Stage::Maintenance, CALL_TIMEOUT);
|
||||
let context = session.context(Stage::Maintenance, session.budget(Stage::Maintenance));
|
||||
let instance_id = context.instance_id.clone();
|
||||
let resource_scope_id = context.resource_scope_id.clone();
|
||||
let budget_ms = context.timeout_ms;
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
() = cancellation.cancelled() => return,
|
||||
() = cancellation.cancelled() => {
|
||||
tracing::info!(target: "plugin.maintenance", %instance_id, %resource_scope_id, budget_ms,
|
||||
budget_source = ?session.budget_source(Stage::Maintenance),
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
reason = "cancelled", "plugin maintenance ended");
|
||||
return;
|
||||
},
|
||||
result = session.call("plugin.reconcile", context, serde_json::json!({}), Vec::new()) => result,
|
||||
};
|
||||
if result.is_err() {
|
||||
tracing::warn!(%instance_id, "plugin maintenance failed; retrying after bounded delay");
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(target: "plugin.maintenance", %instance_id, %resource_scope_id, budget_ms,
|
||||
budget_source = ?session.budget_source(Stage::Maintenance),
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
reason = error.diagnostic().0, "plugin maintenance failed; retrying after bounded delay");
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = cancellation.cancelled() => return,
|
||||
() = tokio::time::sleep(RETRY_DELAY) => {},
|
||||
}
|
||||
dirty.notify_one();
|
||||
} else {
|
||||
tracing::info!(target: "plugin.maintenance", %instance_id, %resource_scope_id, budget_ms,
|
||||
budget_source = ?session.budget_source(Stage::Maintenance),
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
reason = "completed", "plugin maintenance ended");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,8 @@ use super::{
|
||||
use crate::{
|
||||
PackageLimits, RpcLimits, RpcSession,
|
||||
callback::{
|
||||
PluginAccountPortSlot, PluginAffinityPortSlot, PluginClientKeyPortSlot, PluginModelPortSlot,
|
||||
PluginAccountPortSlot, PluginAffinityPortSlot, PluginClientIdentityPortSlot,
|
||||
PluginClientKeyPortSlot, PluginModelPortSlot,
|
||||
},
|
||||
};
|
||||
use gateway_admin::{
|
||||
@@ -21,21 +22,19 @@ use gateway_admin::{
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
authentication::FrontendAuthenticationExtensionIndex,
|
||||
authentication::{ClientIdentityPort, FrontendAuthenticationExtensionIndex},
|
||||
extensions::ExecutionExtensionIndex,
|
||||
nested::{AffinityLookupPort, NestedModelExecutionPort},
|
||||
observation::RequestObserverExtensionIndex,
|
||||
policy::RequestPolicyExtensionIndex,
|
||||
},
|
||||
lifecycle::CancellationToken,
|
||||
provider_ports::OAuthPendingFlowPort,
|
||||
};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
Arc, Mutex as SyncMutex, Weak,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
sync::{Arc, Mutex as SyncMutex, Weak},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::sync::{Mutex, Semaphore};
|
||||
@@ -62,7 +61,7 @@ pub struct PluginRuntime {
|
||||
pub(super) prepared_instances: Mutex<BTreeMap<String, Weak<PreparedInstance>>>,
|
||||
pub(super) preparation_diagnostics: SyncMutex<BTreeMap<u64, PreparationDiagnostic>>,
|
||||
pub(super) restart_circuits: Arc<RestartCircuits>,
|
||||
pub(super) shutting_down: Arc<AtomicBool>,
|
||||
pub(super) shutting_down: CancellationToken,
|
||||
shutdown_lock: Mutex<()>,
|
||||
pub(super) processes: Arc<gateway_host::process::ProcessSupervisor>,
|
||||
pub(super) validators: Arc<Semaphore>,
|
||||
@@ -71,6 +70,7 @@ pub struct PluginRuntime {
|
||||
pub(super) account_ports: Arc<PluginAccountPortSlot>,
|
||||
pub(super) client_key_ports: Arc<PluginClientKeyPortSlot>,
|
||||
pub(super) resource_ports: Arc<crate::callback::PluginResourcePorts>,
|
||||
pub(super) client_identity_port: Arc<PluginClientIdentityPortSlot>,
|
||||
pub(super) model_ports: Arc<PluginModelPortSlot>,
|
||||
pub(super) affinity_ports: Arc<PluginAffinityPortSlot>,
|
||||
pub(super) observers: RequestObserverExtensionIndex,
|
||||
@@ -105,7 +105,7 @@ impl PluginRuntime {
|
||||
prepared_instances: Mutex::new(BTreeMap::new()),
|
||||
preparation_diagnostics: SyncMutex::new(BTreeMap::new()),
|
||||
restart_circuits: RestartCircuits::new(restart_circuit),
|
||||
shutting_down: Arc::new(AtomicBool::new(false)),
|
||||
shutting_down: CancellationToken::new(),
|
||||
shutdown_lock: Mutex::new(()),
|
||||
processes,
|
||||
validators: Arc::new(Semaphore::new(MAXIMUM_PARALLEL_PREPARATIONS)),
|
||||
@@ -115,6 +115,7 @@ impl PluginRuntime {
|
||||
account_ports: Arc::new(PluginAccountPortSlot::new()),
|
||||
client_key_ports: Arc::new(PluginClientKeyPortSlot::new()),
|
||||
resource_ports: Arc::new(crate::callback::PluginResourcePorts::new()),
|
||||
client_identity_port: Arc::new(PluginClientIdentityPortSlot::new()),
|
||||
model_ports: Arc::new(PluginModelPortSlot::new()),
|
||||
affinity_ports: Arc::new(PluginAffinityPortSlot::new()),
|
||||
observers: RequestObserverExtensionIndex::default(),
|
||||
@@ -170,6 +171,14 @@ impl PluginRuntime {
|
||||
self.client_key_ports.bind(access)
|
||||
}
|
||||
|
||||
/// 身份查询独立装配,复用 Core 的认证与冻结权限,不要求绑定模型执行端口
|
||||
pub fn bind_client_identity_port(
|
||||
&self,
|
||||
clients: &Arc<dyn ClientIdentityPort>,
|
||||
) -> Result<(), AdminError> {
|
||||
self.client_identity_port.bind(clients)
|
||||
}
|
||||
|
||||
/// Core 激活后一次性绑定嵌套执行与亲和查询端口;Runtime 仅保存 Weak,避免组合根强环
|
||||
pub fn bind_model_ports(
|
||||
&self,
|
||||
@@ -182,7 +191,7 @@ impl PluginRuntime {
|
||||
|
||||
/// 调用方必须先停止接收 HTTP 与 Worker;这里拒绝新候选并等待全部插件 I/O 退出
|
||||
pub async fn shutdown(&self) {
|
||||
self.shutting_down.store(true, Ordering::Release);
|
||||
self.shutting_down.cancel();
|
||||
let _shutdown = self.shutdown_lock.lock().await;
|
||||
let _prepare = self.prepare_lock.lock().await;
|
||||
let sets = {
|
||||
|
||||
@@ -10,18 +10,12 @@ use gateway_core::{
|
||||
extensions::ExecutionExtensionPlans, observation::RequestObserverPlan,
|
||||
policy::RequestPolicyPlan,
|
||||
},
|
||||
lifecycle::CancellationToken,
|
||||
routing::extensions::{ExtensionSetId, ExtensionSetLease},
|
||||
};
|
||||
use secrecy::ExposeSecret as _;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Duration};
|
||||
|
||||
pub(super) struct PreparedSet {
|
||||
pub(super) id: ExtensionSetId,
|
||||
@@ -32,7 +26,7 @@ pub(super) struct PreparedSet {
|
||||
Option<Arc<dyn gateway_core::engine::authentication::FrontendAuthenticationPlan>>,
|
||||
pub(super) sessions: Vec<Arc<PreparedInstance>>,
|
||||
pub(super) failures: BTreeMap<String, PluginInstanceRuntimeFailure>,
|
||||
pub(super) shutting_down: Arc<AtomicBool>,
|
||||
pub(super) shutting_down: CancellationToken,
|
||||
pub(super) commands: Vec<Arc<crate::adapter::command_line::PluginCommand>>,
|
||||
pub(super) management: Vec<crate::adapter::management::ManagementEntry>,
|
||||
pub(super) model_aliases: Vec<gateway_core::routing::ContributedModelAlias>,
|
||||
@@ -94,7 +88,7 @@ impl ExtensionSetLease for PreparedSet {
|
||||
|
||||
fn can_serve(&self) -> bool {
|
||||
// 发布的是可用能力和故障绑定组成的完整计划;单个进程退出不能使原生转发失效
|
||||
!self.shutting_down.load(Ordering::Acquire)
|
||||
!self.shutting_down.is_cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,8 +26,7 @@ use gateway_core::{
|
||||
};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
sync::{Arc, Weak, atomic::Ordering},
|
||||
time::Duration,
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
impl PluginRuntime {
|
||||
@@ -39,9 +38,11 @@ impl PluginRuntime {
|
||||
) -> Result<ExtensionSetReference, AdminError> {
|
||||
let target_revision = snapshot.config_revision.get();
|
||||
self.record_preparation(target_revision, PreparationDiagnostic::Preparing);
|
||||
let result = self
|
||||
.prepare_inner(snapshot, required_revision, request_body_limit)
|
||||
.await;
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
() = self.shutting_down.cancelled() => Err(AdminError::unavailable("插件运行时正在关闭")),
|
||||
result = self.prepare_inner(snapshot, required_revision, request_body_limit) => result,
|
||||
};
|
||||
match &result {
|
||||
Ok(reference) => self.record_preparation(
|
||||
target_revision,
|
||||
@@ -63,7 +64,7 @@ impl PluginRuntime {
|
||||
required_revision: Option<Revision>,
|
||||
request_body_limit: Option<usize>,
|
||||
) -> Result<ExtensionSetReference, AdminError> {
|
||||
if self.shutting_down.load(Ordering::Acquire) {
|
||||
if self.shutting_down.is_cancelled() {
|
||||
return Err(AdminError::unavailable("插件运行时正在关闭"));
|
||||
}
|
||||
if snapshot.instances.len() > 64 {
|
||||
@@ -72,7 +73,7 @@ impl PluginRuntime {
|
||||
let fingerprint = fingerprint(&snapshot)?;
|
||||
// 准备串行化与弱索引分锁;诊断和已发布请求不会等待外部 I/O
|
||||
let _prepare = self.prepare_lock.lock().await;
|
||||
if self.shutting_down.load(Ordering::Acquire) {
|
||||
if self.shutting_down.is_cancelled() {
|
||||
return Err(AdminError::unavailable("插件运行时正在关闭"));
|
||||
}
|
||||
let existing = {
|
||||
@@ -125,17 +126,12 @@ impl PluginRuntime {
|
||||
let result = if let Some(instance) = reusable {
|
||||
Ok(instance)
|
||||
} else {
|
||||
tokio::time::timeout(
|
||||
// 沿用准备阶段原有的 30 秒预算;打包校验与能力恢复也在预算内
|
||||
Duration::from_secs(30),
|
||||
self.prepare_instance(
|
||||
instance.clone(),
|
||||
snapshot.config_revision,
|
||||
request_body_limit,
|
||||
),
|
||||
self.prepare_instance(
|
||||
instance.clone(),
|
||||
snapshot.config_revision,
|
||||
request_body_limit,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(AdminError::unavailable("插件启动超时")))
|
||||
};
|
||||
(instance, identity, result)
|
||||
},
|
||||
@@ -208,7 +204,6 @@ impl PluginRuntime {
|
||||
.rpc_limits
|
||||
.maximum_calls
|
||||
.min(self.config.rpc_limits.maximum_executing_calls),
|
||||
self.config.rpc_limits.maximum_call_timeout,
|
||||
self.config.rpc_limits.maximum_buffered_body_bytes,
|
||||
)
|
||||
.map(|plan| {
|
||||
@@ -217,21 +212,16 @@ impl PluginRuntime {
|
||||
.map_err(|_| AdminError::invalid("插件观察计划注册冲突"))
|
||||
})
|
||||
.transpose()?;
|
||||
let policies = crate::adapter::policy::PluginRequestPolicyPlan::compile(
|
||||
policy_entries,
|
||||
self.config.rpc_limits.maximum_call_timeout,
|
||||
)?
|
||||
.map(|plan| {
|
||||
self.policies
|
||||
.register(id.clone(), plan)
|
||||
.map_err(|_| AdminError::invalid("插件请求策略计划注册冲突"))
|
||||
})
|
||||
.transpose()?;
|
||||
let middleware = crate::adapter::middleware::PluginMiddlewarePlan::compile(
|
||||
middleware_entries,
|
||||
self.config.rpc_limits.maximum_call_timeout,
|
||||
)
|
||||
.map(|plan| plan as Arc<dyn gateway_core::engine::middleware::MiddlewarePlan>);
|
||||
let policies = crate::adapter::policy::PluginRequestPolicyPlan::compile(policy_entries)?
|
||||
.map(|plan| {
|
||||
self.policies
|
||||
.register(id.clone(), plan)
|
||||
.map_err(|_| AdminError::invalid("插件请求策略计划注册冲突"))
|
||||
})
|
||||
.transpose()?;
|
||||
let middleware =
|
||||
crate::adapter::middleware::PluginMiddlewarePlan::compile(middleware_entries)
|
||||
.map(|plan| plan as Arc<dyn gateway_core::engine::middleware::MiddlewarePlan>);
|
||||
let upstream_adapters =
|
||||
crate::adapter::upstream_adapter::PluginUpstreamAdapterPlan::compile(upstream_entries);
|
||||
let execution = if middleware.is_some() || upstream_adapters.is_some() {
|
||||
@@ -252,7 +242,6 @@ impl PluginRuntime {
|
||||
let authentication =
|
||||
crate::adapter::frontend_authentication::PluginFrontendAuthenticationPlan::compile(
|
||||
authentication_entries,
|
||||
self.config.rpc_limits.maximum_call_timeout,
|
||||
)?
|
||||
.map(|plan| {
|
||||
self.authentications
|
||||
|
||||
@@ -90,9 +90,10 @@ impl PluginRuntime {
|
||||
.session
|
||||
.call(
|
||||
"plugin.state.migrate",
|
||||
instance
|
||||
.session
|
||||
.context(Stage::Configuration, Duration::from_secs(30)),
|
||||
instance.session.context(
|
||||
Stage::Configuration,
|
||||
instance.session.budget(Stage::Configuration),
|
||||
),
|
||||
serde_json::json!({}),
|
||||
serde_json::to_vec(&request)
|
||||
.map_err(|_| AdminError::internal("状态迁移批次无法编码"))?,
|
||||
@@ -207,7 +208,11 @@ impl gateway_admin::ports::plugins::PluginStateLifecycle for PluginRuntime {
|
||||
prepared: &ExtensionSetReference,
|
||||
transition: PluginStateTransition,
|
||||
) -> Result<(), AdminError> {
|
||||
self.run_state_migration(prepared, transition).await
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = self.shutting_down.cancelled() => Err(AdminError::unavailable("插件运行时正在关闭")),
|
||||
result = self.run_state_migration(prepared, transition) => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(super) fn map_state_admin_error(
|
||||
|
||||
@@ -47,4 +47,11 @@ impl ExecutionCapacity {
|
||||
self.active.close();
|
||||
self.queued.close();
|
||||
}
|
||||
|
||||
pub(super) fn available(&self) -> (usize, usize) {
|
||||
(
|
||||
self.active.available_permits(),
|
||||
self.queued.available_permits(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
//! RPC 调用表唯一持有调度、回调、响应流和取消确认,统一完成调用资源回收
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use gateway_host::process::ProcessControl;
|
||||
use gateway_plugin_sdk::{
|
||||
CallContext, Frame, Message, PluginFault,
|
||||
client::{CallDeadline, CallDeadlinePause, EncodedFrame},
|
||||
};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
|
||||
|
||||
use super::{
|
||||
CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSessionLoad, admission::ExecutionCapacity,
|
||||
};
|
||||
use crate::stream::StreamIngress;
|
||||
|
||||
pub(crate) struct Shared {
|
||||
state: Mutex<State>,
|
||||
started_at: Instant,
|
||||
pub(super) expected_stop: AtomicBool,
|
||||
failure_observer: Option<Arc<dyn Fn(Duration) + Send + Sync>>,
|
||||
pub control: mpsc::Sender<EncodedFrame>,
|
||||
pub process: ProcessControl,
|
||||
pub stopped: watch::Sender<bool>,
|
||||
callback_slots: Arc<Semaphore>,
|
||||
pub(super) execution: ExecutionCapacity,
|
||||
cancellations_changed: Notify,
|
||||
}
|
||||
|
||||
struct State {
|
||||
failure: Option<RpcError>,
|
||||
calls: BTreeMap<u64, PendingCall>,
|
||||
}
|
||||
|
||||
struct PendingCall {
|
||||
_slot: Arc<OwnedSemaphorePermit>,
|
||||
execution: Option<OwnedSemaphorePermit>,
|
||||
deadline: CallDeadline,
|
||||
handler: Arc<dyn CallbackHandler>,
|
||||
transmitted: bool,
|
||||
cancellation: Option<RpcError>,
|
||||
context: CallContext,
|
||||
result: Option<oneshot::Sender<Result<RpcReply, RpcError>>>,
|
||||
stream: Option<StreamIngress>,
|
||||
scheduling: Option<Scheduling>,
|
||||
callbacks: BTreeMap<u64, CallbackTask>,
|
||||
}
|
||||
|
||||
// 调度状态只属于父任务;回调完成与恢复授权分别推进
|
||||
enum Scheduling {
|
||||
Running {
|
||||
epoch: u64,
|
||||
},
|
||||
Suspended {
|
||||
epoch: u64,
|
||||
pause: Option<CallDeadlinePause>,
|
||||
},
|
||||
Resuming {
|
||||
epoch: u64,
|
||||
pause: Option<CallDeadlinePause>,
|
||||
task: tokio::task::AbortHandle,
|
||||
},
|
||||
}
|
||||
|
||||
struct CallbackTask {
|
||||
task: tokio::task::AbortHandle,
|
||||
pauses_budget: bool,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
impl Drop for CallbackTask {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl PendingCall {
|
||||
/// 关闭资源与封存副作用必须先于任何结果通知;取消确认只延续容量租约
|
||||
fn revoke(&mut self) {
|
||||
self.handler.finish(&self.context);
|
||||
self.callbacks.clear();
|
||||
self.stop_resumption();
|
||||
}
|
||||
|
||||
fn stop_resumption(&mut self) {
|
||||
if let Some(Scheduling::Resuming { task, .. }) = self.scheduling.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(mut self, result: Result<RpcReply, RpcError>) {
|
||||
tracing::debug!(
|
||||
event = "plugin.rpc.finished",
|
||||
instance_id = self.context.instance_id,
|
||||
call_id = self.context.call_id,
|
||||
stage = ?self.context.stage,
|
||||
outcome = result.as_ref().map_or_else(|error| error.diagnostic().0, |_| "success"),
|
||||
"插件 RPC 调用结束"
|
||||
);
|
||||
let stream = self.stream.take();
|
||||
let sender = self.result.take();
|
||||
// 先同步关闭调用资源,Core 才能在收到结果时看到全部发送副作用
|
||||
drop(self);
|
||||
if let Some(stream) = stream {
|
||||
stream.finish(result.as_ref().map(|_| ()).map_err(Clone::clone));
|
||||
}
|
||||
if let Some(sender) = sender {
|
||||
let _ = sender.send(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingCall {
|
||||
fn drop(&mut self) {
|
||||
self.revoke();
|
||||
}
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
pub(super) fn new(
|
||||
limits: RpcLimits,
|
||||
process: ProcessControl,
|
||||
control: mpsc::Sender<EncodedFrame>,
|
||||
failure_observer: Option<Arc<dyn Fn(Duration) + Send + Sync>>,
|
||||
) -> Self {
|
||||
let (stopped, _) = watch::channel(false);
|
||||
Self {
|
||||
state: Mutex::new(State {
|
||||
failure: None,
|
||||
calls: BTreeMap::new(),
|
||||
}),
|
||||
started_at: Instant::now(),
|
||||
expected_stop: AtomicBool::new(false),
|
||||
failure_observer,
|
||||
control,
|
||||
process,
|
||||
stopped,
|
||||
callback_slots: Arc::new(Semaphore::new(limits.maximum_callbacks)),
|
||||
execution: ExecutionCapacity::new(
|
||||
limits.maximum_executing_calls,
|
||||
limits.maximum_queued_calls,
|
||||
limits.maximum_queue_timeout,
|
||||
),
|
||||
cancellations_changed: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn failure_diagnostic(&self) -> Option<(&'static str, &'static str)> {
|
||||
self.state().failure.as_ref().map(RpcError::diagnostic)
|
||||
}
|
||||
|
||||
pub(super) fn load(&self, limits: RpcLimits, available_calls: usize) -> RpcSessionLoad {
|
||||
let state = self.state();
|
||||
let (available_execution, available_queue) = self.execution.available();
|
||||
RpcSessionLoad {
|
||||
active_calls: limits.maximum_calls - available_calls,
|
||||
executing_calls: limits.maximum_executing_calls - available_execution,
|
||||
queued_calls: limits.maximum_queued_calls - available_queue,
|
||||
resuming_calls: state
|
||||
.calls
|
||||
.values()
|
||||
.filter(|call| matches!(call.scheduling, Some(Scheduling::Resuming { .. })))
|
||||
.count(),
|
||||
pending_callbacks: limits.maximum_callbacks - self.callback_slots.available_permits(),
|
||||
active_streams: state
|
||||
.calls
|
||||
.values()
|
||||
.filter(|call| call.stream.is_some() && call.result.is_none())
|
||||
.count(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn begin_call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
deadline: tokio::time::Instant,
|
||||
stream: Option<StreamIngress>,
|
||||
slot: Arc<OwnedSemaphorePermit>,
|
||||
execution: OwnedSemaphorePermit,
|
||||
handler: Arc<dyn CallbackHandler>,
|
||||
) -> Result<StartedCall, RpcError> {
|
||||
let id = context.call_id;
|
||||
let (result, received) = oneshot::channel();
|
||||
let mut state = self.state();
|
||||
if let Some(error) = &state.failure {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let deadline = CallDeadline::new(deadline);
|
||||
handler.begin(&context, &deadline);
|
||||
state.calls.insert(
|
||||
id,
|
||||
PendingCall {
|
||||
_slot: slot,
|
||||
execution: Some(execution),
|
||||
deadline: deadline.clone(),
|
||||
handler,
|
||||
transmitted: false,
|
||||
cancellation: None,
|
||||
context,
|
||||
result: Some(result),
|
||||
stream,
|
||||
scheduling: Some(Scheduling::Running { epoch: 0 }),
|
||||
callbacks: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
Ok(StartedCall {
|
||||
id,
|
||||
deadline,
|
||||
received,
|
||||
})
|
||||
}
|
||||
|
||||
/// 替换维护任务前等待已取消的调用确认;未确认仍沿用会话的有界故障回收
|
||||
pub(crate) async fn settle_cancellations(&self) {
|
||||
let ids: Vec<_> = self
|
||||
.state()
|
||||
.calls
|
||||
.iter()
|
||||
.filter_map(|(id, call)| call.cancellation.is_some().then_some(*id))
|
||||
.collect();
|
||||
loop {
|
||||
let changed = self.cancellations_changed.notified();
|
||||
tokio::pin!(changed);
|
||||
changed.as_mut().enable();
|
||||
if !ids.iter().any(|id| self.is_cancelling(*id)) {
|
||||
break;
|
||||
}
|
||||
changed.await;
|
||||
}
|
||||
if self.failure_diagnostic().is_some() {
|
||||
self.process.exited().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn state(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub fn fail(&self, error: RpcError) {
|
||||
let uptime = self.started_at.elapsed();
|
||||
let (calls, failure_observer) = {
|
||||
let mut state = self.state();
|
||||
if state.failure.is_some() {
|
||||
return;
|
||||
}
|
||||
state.failure = Some(error.clone());
|
||||
let observer = (!self.expected_stop.load(Ordering::Acquire))
|
||||
.then(|| self.failure_observer.clone())
|
||||
.flatten();
|
||||
(std::mem::take(&mut state.calls), observer)
|
||||
};
|
||||
self.execution.close();
|
||||
for call in calls.into_values() {
|
||||
call.finish(Err(error.clone()));
|
||||
}
|
||||
if let Some(observer) = failure_observer {
|
||||
observer(uptime);
|
||||
}
|
||||
self.stopped.send_replace(true);
|
||||
self.process.stop();
|
||||
self.cancellations_changed.notify_waiters();
|
||||
}
|
||||
|
||||
pub(super) fn expect_stop(&self) {
|
||||
self.expected_stop.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn finish(&self, id: u64, result: Result<RpcReply, RpcError>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if matches!(&result, Err(RpcError::Remote(fault)) if fault.code == gateway_plugin_sdk::ErrorCode::Timeout)
|
||||
{
|
||||
log_timeout(&call.context, "plugin", "initial_reply", "remote_timeout");
|
||||
}
|
||||
if result.is_ok() && !matches!(call.scheduling, Some(Scheduling::Running { .. })) {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
if call.stream.is_some() && result.is_ok() {
|
||||
if call.context.resource_stream {
|
||||
call.deadline.clear();
|
||||
}
|
||||
let initial = call.result.take().ok_or(RpcError::Protocol(None))?;
|
||||
call.execution.take();
|
||||
call.scheduling.take();
|
||||
call.handler.streaming(&call.context);
|
||||
drop(state);
|
||||
let _ = initial.send(result);
|
||||
} else {
|
||||
let pending = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
|
||||
drop(state);
|
||||
pending.finish(result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stream_chunk(&self, id: u64, sequence: u64, payload: Vec<u8>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if call.result.is_some() {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
call.stream
|
||||
.as_mut()
|
||||
.ok_or(RpcError::Protocol(None))?
|
||||
.receive(sequence, payload)
|
||||
}
|
||||
|
||||
pub fn stream_end(&self, id: u64, error: Option<PluginFault>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
if state
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let call = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
|
||||
drop(state);
|
||||
if call.stream.is_none() || call.result.is_some() {
|
||||
call.finish(Err(RpcError::Protocol(None)));
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let result = error.map_or_else(
|
||||
|| {
|
||||
Ok(RpcReply {
|
||||
result: serde_json::Value::Null,
|
||||
payload: Vec::new(),
|
||||
})
|
||||
},
|
||||
|error| Err(RpcError::Remote(error)),
|
||||
);
|
||||
call.finish(result);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn release_stream_credit(&self, id: u64, bytes: u32) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
// 完成后仍可消费已排队数据,无须再给已关闭的流发信用
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return Ok(());
|
||||
};
|
||||
call.stream
|
||||
.as_mut()
|
||||
.ok_or(RpcError::Protocol(None))?
|
||||
.release(bytes)?;
|
||||
drop(state);
|
||||
self.send_control(Frame::control(Message::Credit {
|
||||
id,
|
||||
bytes,
|
||||
frames: 1,
|
||||
}));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn context(&self, id: u64) -> Option<CallContext> {
|
||||
self.state()
|
||||
.calls
|
||||
.get(&id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
.map(|call| call.context.clone())
|
||||
}
|
||||
|
||||
pub fn is_cancelling(&self, id: u64) -> bool {
|
||||
self.state()
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
}
|
||||
|
||||
pub fn mark_transmitted(&self, id: u64) -> bool {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return false;
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return false;
|
||||
}
|
||||
call.transmitted = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// 先撤销调用权限,再等待对端确认;只有无响应对端才终止整个 incarnation
|
||||
pub fn cancel(self: &Arc<Self>, id: u64, error: RpcError) {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return;
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return;
|
||||
}
|
||||
if matches!(error, RpcError::Timeout) {
|
||||
log_timeout(
|
||||
&call.context,
|
||||
"runtime",
|
||||
if call.result.is_some() {
|
||||
"initial_reply"
|
||||
} else {
|
||||
"response_stream"
|
||||
},
|
||||
"processing_budget_exhausted",
|
||||
);
|
||||
}
|
||||
if !call.transmitted {
|
||||
let call = state.calls.remove(&id);
|
||||
drop(state);
|
||||
if let Some(call) = call {
|
||||
call.finish(Err(error));
|
||||
}
|
||||
return;
|
||||
}
|
||||
call.cancellation = Some(error.clone());
|
||||
// 同步撤销不能跨越取消确认:收到确认的并发任务可能立即复用实例
|
||||
// CallbackHandler::finish 合同要求非阻塞;真正的等待与结果交付放在锁外
|
||||
call.revoke();
|
||||
let result = call.result.take();
|
||||
let stream = call.stream.take();
|
||||
tracing::debug!(
|
||||
event = "plugin.rpc.cancelled",
|
||||
instance_id = call.context.instance_id,
|
||||
call_id = id,
|
||||
stage = ?call.context.stage,
|
||||
reason = error.diagnostic().0,
|
||||
"插件 RPC 调用已撤销,等待进程确认"
|
||||
);
|
||||
drop(state);
|
||||
if let Some(stream) = stream {
|
||||
stream.finish(Err(error.clone()));
|
||||
}
|
||||
if let Some(result) = result {
|
||||
let _ = result.send(Err(error.clone()));
|
||||
}
|
||||
self.send_control(Frame::control(Message::Cancel { id }));
|
||||
let shared = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
if shared.is_cancelling(id) {
|
||||
shared.fail(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acknowledge_cancellation(&self, id: u64) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
if !state
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
{
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let call = state.calls.remove(&id);
|
||||
drop(state);
|
||||
drop(call);
|
||||
self.cancellations_changed.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn track_callback(
|
||||
&self,
|
||||
parent_id: u64,
|
||||
id: u64,
|
||||
method: &str,
|
||||
task: tokio::task::AbortHandle,
|
||||
permit: OwnedSemaphorePermit,
|
||||
) -> bool {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state
|
||||
.calls
|
||||
.get_mut(&parent_id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
else {
|
||||
task.abort();
|
||||
return false;
|
||||
};
|
||||
let pauses_budget = method == gateway_plugin_sdk::call::middleware::NEXT_METHOD;
|
||||
if !pauses_budget
|
||||
&& let Some(Scheduling::Suspended { pause, .. } | Scheduling::Resuming { pause, .. }) =
|
||||
call.scheduling.as_mut()
|
||||
{
|
||||
pause.take();
|
||||
}
|
||||
call.callbacks.insert(
|
||||
id,
|
||||
CallbackTask {
|
||||
task,
|
||||
pauses_budget,
|
||||
_permit: permit,
|
||||
},
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn consume_callback(&self, parent_id: u64, id: u64) -> Result<(), RpcError> {
|
||||
if id == 0 || !id.is_multiple_of(2) || parent_id == 0 || parent_id.is_multiple_of(2) {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
// 取消确认可以与消费确认交错;确认只能撤销资源,不能重新创建调用
|
||||
if let Some(call) = self.state().calls.get_mut(&parent_id) {
|
||||
call.callbacks.remove(&id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn suspend(&self, id: u64, epoch: u64) -> Result<(), RpcError> {
|
||||
if id == 0 || id.is_multiple_of(2) || epoch == 0 {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return Ok(());
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
match call.scheduling.as_ref() {
|
||||
Some(Scheduling::Running { epoch: previous })
|
||||
if previous.checked_add(1) == Some(epoch) => {}
|
||||
_ => return Err(RpcError::Protocol(None)),
|
||||
}
|
||||
let pauses = !call.callbacks.is_empty()
|
||||
&& call
|
||||
.callbacks
|
||||
.values()
|
||||
.all(|callback| callback.pauses_budget);
|
||||
let pause = pauses.then(|| call.deadline.pause()).flatten();
|
||||
call.execution.take();
|
||||
call.scheduling = Some(Scheduling::Suspended { epoch, pause });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn request_resume(self: &Arc<Self>, id: u64, epoch: u64) -> Result<(), RpcError> {
|
||||
if id == 0 || id.is_multiple_of(2) || epoch == 0 {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return Ok(());
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if !matches!(call.scheduling, Some(Scheduling::Suspended { epoch: current, .. }) if current == epoch)
|
||||
{
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let Some(Scheduling::Suspended { pause, .. }) = call.scheduling.take() else {
|
||||
return Err(RpcError::Protocol(None));
|
||||
};
|
||||
let shared = Arc::clone(self);
|
||||
let (ready, started) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
if started.await.is_err() {
|
||||
return;
|
||||
}
|
||||
match shared.execution.resume().await {
|
||||
Ok(permit) => shared.grant_resume(id, epoch, permit),
|
||||
Err(error) => shared.cancel(id, error),
|
||||
}
|
||||
});
|
||||
call.scheduling = Some(Scheduling::Resuming {
|
||||
epoch,
|
||||
pause,
|
||||
task: task.abort_handle(),
|
||||
});
|
||||
let _ = ready.send(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn grant_resume(&self, id: u64, epoch: u64, permit: OwnedSemaphorePermit) {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state
|
||||
.calls
|
||||
.get_mut(&id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if !matches!(call.scheduling, Some(Scheduling::Resuming { epoch: current, .. }) if current == epoch)
|
||||
{
|
||||
return;
|
||||
}
|
||||
call.execution = Some(permit);
|
||||
call.scheduling = Some(Scheduling::Running { epoch });
|
||||
drop(state);
|
||||
self.send_control(Frame::control(Message::Resumed { id, epoch }));
|
||||
}
|
||||
|
||||
pub fn send_control(&self, frame: Frame) {
|
||||
let Ok(frame) = EncodedFrame::new(frame) else {
|
||||
self.fail(RpcError::Protocol(None));
|
||||
return;
|
||||
};
|
||||
self.send_encoded_control(frame);
|
||||
}
|
||||
|
||||
pub(super) fn send_encoded_control(&self, frame: EncodedFrame) {
|
||||
if self.control.try_send(frame).is_err() {
|
||||
self.fail(RpcError::Capacity);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_callback_slot(&self) -> Option<OwnedSemaphorePermit> {
|
||||
self.callback_slots.clone().try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_callbacks(&self, limits: RpcLimits) {
|
||||
if let Ok(permits) = self
|
||||
.callback_slots
|
||||
.clone()
|
||||
.acquire_many_owned(limits.maximum_callbacks as u32)
|
||||
.await
|
||||
{
|
||||
drop(permits);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct StartedCall {
|
||||
pub(super) id: u64,
|
||||
pub(super) deadline: CallDeadline,
|
||||
pub(super) received: oneshot::Receiver<Result<RpcReply, RpcError>>,
|
||||
}
|
||||
|
||||
fn log_timeout(
|
||||
context: &CallContext,
|
||||
owner: &'static str,
|
||||
phase: &'static str,
|
||||
reason: &'static str,
|
||||
) {
|
||||
tracing::warn!(
|
||||
event = "plugin.rpc.timeout",
|
||||
owner,
|
||||
phase,
|
||||
instance_id = context.instance_id,
|
||||
call_id = context.call_id,
|
||||
request_id = context.request_id,
|
||||
attempt_id = context.attempt_id,
|
||||
stage = ?context.stage,
|
||||
budget_ms = context.timeout_ms,
|
||||
reason,
|
||||
"插件 RPC 调用超时"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
//! RPC 会话的容量、错误、回调与生命周期合同
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_host::process::ProcessStartError;
|
||||
use gateway_plugin_sdk::{CallBudgetError, CallContext, PluginFault, Stage, client::CallDeadline};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RpcLimits {
|
||||
/// 受管回调与观察队列的缓冲预算,不是请求正文或 IPC 消息总量上限
|
||||
pub maximum_buffered_body_bytes: usize,
|
||||
/// 包含初次排队、下游等待、取消确认和未消费流的总在途上限
|
||||
pub maximum_calls: usize,
|
||||
pub maximum_executing_calls: usize,
|
||||
pub maximum_queued_calls: usize,
|
||||
pub maximum_queue_timeout: Duration,
|
||||
pub maximum_callbacks: usize,
|
||||
pub handshake_timeout: Duration,
|
||||
pub maximum_stderr_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for RpcLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
maximum_buffered_body_bytes: 1024 * 1024,
|
||||
maximum_calls: 256,
|
||||
maximum_executing_calls: 64,
|
||||
maximum_queued_calls: 64,
|
||||
maximum_queue_timeout: Duration::from_secs(30),
|
||||
maximum_callbacks: 320,
|
||||
handshake_timeout: Duration::from_secs(5),
|
||||
maximum_stderr_bytes: 64 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub enum RpcError {
|
||||
#[error("plugin process could not be started: {0}")]
|
||||
Start(#[source] ProcessStartError),
|
||||
#[error("plugin handshake does not match the prepared instance")]
|
||||
Handshake(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin call budget negotiation failed: {0}")]
|
||||
Budget(#[from] CallBudgetError),
|
||||
#[error("plugin protocol is invalid")]
|
||||
Protocol(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin returned an invalid response at {0:?}")]
|
||||
InvalidResponse(Stage, #[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin process or transport has stopped")]
|
||||
Closed(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin call exceeded its deadline")]
|
||||
Timeout,
|
||||
#[error("plugin session was cancelled")]
|
||||
Cancelled,
|
||||
#[error("plugin call capacity is exhausted")]
|
||||
Capacity,
|
||||
#[error("plugin admission capacity is exhausted")]
|
||||
Overloaded,
|
||||
#[error("plugin call context is invalid")]
|
||||
Context(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin returned an error")]
|
||||
Remote(PluginFault),
|
||||
}
|
||||
|
||||
impl RpcError {
|
||||
pub(crate) fn is_overloaded(&self) -> bool {
|
||||
matches!(self, Self::Overloaded)
|
||||
|| matches!(self, Self::Remote(fault) if fault.code == gateway_plugin_sdk::ErrorCode::Capacity)
|
||||
}
|
||||
|
||||
/// 会话状态与请求错误共用安全分类,不展开插件返回正文
|
||||
pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) {
|
||||
match self {
|
||||
RpcError::Start(_) => ("process_start", "插件进程启动失败"),
|
||||
RpcError::Handshake(_) => ("handshake", "插件握手失败"),
|
||||
RpcError::Budget(_) => ("call_budget", "插件调用预算协商失败"),
|
||||
RpcError::Protocol(_) => ("protocol", "插件协议错误"),
|
||||
RpcError::InvalidResponse(stage, _) => (
|
||||
"invalid_response",
|
||||
match stage {
|
||||
Stage::Registration => "注册阶段:插件返回了无效响应",
|
||||
Stage::Configuration => "配置阶段:插件返回了无效响应",
|
||||
Stage::Authentication => "认证阶段:插件返回了无效响应",
|
||||
Stage::Routing => "路由阶段:插件返回了无效响应",
|
||||
Stage::Scheduling => "调度阶段:插件返回了无效响应",
|
||||
Stage::Retry => "重试阶段:插件返回了无效响应",
|
||||
Stage::Http => "HTTP 中间件:插件返回了无效响应",
|
||||
Stage::Service => "服务中间件:插件返回了无效响应",
|
||||
Stage::WebSocket => "WebSocket 中间件:插件返回了无效响应",
|
||||
Stage::Request => "请求中间件:插件返回了无效响应",
|
||||
Stage::Attempt => "尝试中间件:插件返回了无效响应",
|
||||
Stage::Outbound => "出站准备中间件:插件返回了无效响应",
|
||||
Stage::Upstream => "上游适配阶段:插件返回了无效响应",
|
||||
Stage::Observation => "观察阶段:插件返回了无效响应",
|
||||
Stage::Management | Stage::PublicManagement => "管理阶段:插件返回了无效响应",
|
||||
Stage::CommandLine => "命令行阶段:插件返回了无效响应",
|
||||
Stage::Maintenance => "维护阶段:插件返回了无效响应",
|
||||
},
|
||||
),
|
||||
RpcError::Closed(_) => ("closed", "插件进程或传输已停止"),
|
||||
RpcError::Timeout => ("timeout", "插件调用超时"),
|
||||
RpcError::Cancelled => ("cancelled", "插件会话已取消"),
|
||||
RpcError::Capacity => ("capacity", "插件调用容量已耗尽"),
|
||||
RpcError::Overloaded => ("overloaded", "插件暂时繁忙"),
|
||||
RpcError::Context(_) => ("context", "插件调用上下文无效"),
|
||||
RpcError::Remote(_) => ("remote", "插件返回错误"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum RpcSessionDiagnostic {
|
||||
Ready,
|
||||
Quiescing,
|
||||
Failed {
|
||||
code: &'static str,
|
||||
message: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) struct RpcSessionLifecycle {
|
||||
pub(super) failure: Arc<dyn Fn(Duration) + Send + Sync>,
|
||||
end: Option<Box<dyn FnOnce() + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl RpcSessionLifecycle {
|
||||
pub(crate) fn new(
|
||||
failure: impl Fn(Duration) + Send + Sync + 'static,
|
||||
end: impl FnOnce() + Send + Sync + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
failure: Arc::new(failure),
|
||||
end: Some(Box::new(end)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fail(&self, uptime: Duration) {
|
||||
(self.failure)(uptime);
|
||||
}
|
||||
}
|
||||
|
||||
/// 从实际租约与调用表采样,各项存在重叠,不构成同一时刻的可加总分区
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct RpcSessionLoad {
|
||||
/// 总在途租约,包含初次排队、取消确认和 End 后仍由消费者持有的流
|
||||
pub(crate) active_calls: usize,
|
||||
/// 正在占用执行名额的调用,包含取消确认前尚未归还的名额
|
||||
pub(crate) executing_calls: usize,
|
||||
/// 正在占用初次准入排队名额的调用
|
||||
pub(crate) queued_calls: usize,
|
||||
/// 调用表中等待恢复授权的调用
|
||||
pub(crate) resuming_calls: usize,
|
||||
/// 驻留回调名额,包含已回复但插件尚未消费的结果
|
||||
pub(crate) pending_callbacks: usize,
|
||||
/// 已交付首结果且尚未收到 End 或取消的协议响应流
|
||||
pub(crate) active_streams: usize,
|
||||
}
|
||||
|
||||
impl Drop for RpcSessionLifecycle {
|
||||
fn drop(&mut self) {
|
||||
if let Some(end) = self.end.take() {
|
||||
end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不派生 Debug,防止原始请求、凭据或模型输出进入宿主诊断
|
||||
pub struct RpcReply {
|
||||
pub result: serde_json::Value,
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
/// 具体回调在领域适配器校验账号、出站与执行身份;上下文只取自宿主在途调用表
|
||||
pub trait CallbackHandler: Send + Sync {
|
||||
/// 在 Call 可见之前建立资源范围;不得执行 I/O 或调用 RPC
|
||||
fn begin(&self, _context: &CallContext, _deadline: &CallDeadline) {}
|
||||
|
||||
/// 调用结束或取消时立即撤销其句柄;实现必须幂等且不能阻塞
|
||||
fn finish(&self, _context: &CallContext) {}
|
||||
|
||||
/// 首个流式结果已到达;资源仍绑定原调用并在终态统一回收
|
||||
fn streaming(&self, _context: &CallContext) {}
|
||||
|
||||
fn call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
method: String,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> BoxFuture<'static, Result<RpcReply, PluginFault>>;
|
||||
}
|
||||
@@ -4,7 +4,6 @@ use std::sync::Arc;
|
||||
|
||||
use gateway_plugin_sdk::{
|
||||
ErrorCode, Frame, Message, PluginFault,
|
||||
call::middleware::NEXT_METHOD,
|
||||
client::{EncodedFrame, read_frame},
|
||||
};
|
||||
use tokio::{
|
||||
@@ -12,7 +11,7 @@ use tokio::{
|
||||
sync::{mpsc, oneshot},
|
||||
};
|
||||
|
||||
use super::session::{CallbackHandler, RpcError, RpcReply, Shared};
|
||||
use super::{CallbackHandler, RpcError, RpcReply, Shared};
|
||||
|
||||
pub(super) fn start<W, R>(
|
||||
writer: W,
|
||||
@@ -100,6 +99,13 @@ async fn read_loop<R: AsyncRead + Unpin>(
|
||||
}
|
||||
};
|
||||
let result = match frame.message {
|
||||
Message::Suspend { id, epoch } if frame.payload.is_empty() => shared.suspend(id, epoch),
|
||||
Message::Resume { id, epoch } if frame.payload.is_empty() => {
|
||||
shared.request_resume(id, epoch)
|
||||
}
|
||||
Message::CallbackConsumed { id, parent_id } if frame.payload.is_empty() => {
|
||||
shared.consume_callback(parent_id, id)
|
||||
}
|
||||
Message::Cancelled { id } if frame.payload.is_empty() => {
|
||||
shared.acknowledge_cancellation(id)
|
||||
}
|
||||
@@ -125,26 +131,17 @@ async fn read_loop<R: AsyncRead + Unpin>(
|
||||
Err(RpcError::Protocol(None))
|
||||
} else if let Some(context) = shared.context(parent_id) {
|
||||
last_callback = id;
|
||||
if let Some(permit) = shared.try_callback_slot(&method) {
|
||||
if let Some(permit) = shared.try_callback_slot() {
|
||||
let handler = Arc::clone(&callbacks);
|
||||
let response = Arc::clone(&shared);
|
||||
let (ready, started) = oneshot::channel();
|
||||
let pause = (method == NEXT_METHOD)
|
||||
.then(|| shared.pause_next(parent_id))
|
||||
.flatten();
|
||||
let callback_method = method.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
// 先把任务归属登记到父调用,再允许执行宿主操作
|
||||
if started.await.is_err() {
|
||||
return;
|
||||
}
|
||||
let result = handler.call(context, method, params, frame.payload).await;
|
||||
if pause.is_some()
|
||||
&& let Err(error) = response.resume_next(parent_id).await
|
||||
{
|
||||
response.cancel(parent_id, error);
|
||||
return;
|
||||
}
|
||||
drop(pause);
|
||||
let reply = match result {
|
||||
Ok(reply) => Frame {
|
||||
message: Message::Result {
|
||||
@@ -166,9 +163,14 @@ async fn read_loop<R: AsyncRead + Unpin>(
|
||||
),
|
||||
})),
|
||||
}
|
||||
drop(permit);
|
||||
});
|
||||
if shared.track_callback(parent_id, task.abort_handle()) {
|
||||
if shared.track_callback(
|
||||
parent_id,
|
||||
id,
|
||||
&callback_method,
|
||||
task.abort_handle(),
|
||||
permit,
|
||||
) {
|
||||
let _ = ready.send(());
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
//! 插件双向 RPC 会话、调用结果与生命周期的公共入口
|
||||
|
||||
mod admission;
|
||||
mod calls;
|
||||
mod contract;
|
||||
mod dispatch;
|
||||
mod session;
|
||||
|
||||
pub use session::{CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSession};
|
||||
pub(crate) use session::{RpcSessionDiagnostic, RpcSessionLifecycle, Shared};
|
||||
pub(crate) use calls::Shared;
|
||||
pub use contract::{CallbackHandler, RpcError, RpcLimits, RpcReply};
|
||||
pub(crate) use contract::{RpcSessionDiagnostic, RpcSessionLifecycle, RpcSessionLoad};
|
||||
pub use session::RpcSession;
|
||||
|
||||
@@ -1,196 +1,28 @@
|
||||
//! 插件 RPC 会话的调用登记、超时取消、故障传播与关闭管理
|
||||
//! 插件 RPC 会话的握手、对外调用、准入和进程关闭协调
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
sync::{Arc, atomic::Ordering},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_host::process::{ProcessControl, ProcessSpec, ProcessStartError, ProcessSupervisor};
|
||||
use gateway_host::process::{ProcessSpec, ProcessStartError, ProcessSupervisor};
|
||||
use gateway_plugin_sdk::{
|
||||
CallContext, Frame, Handshake, Message, PROTOCOL_VERSION, PluginFault, Stage,
|
||||
client::{CallDeadline, CallDeadlinePause, EncodedFrame, read_frame, write_frame},
|
||||
CallBudgetSource, CallContext, Frame, Handshake, Message, PROTOCOL_VERSION,
|
||||
ResolvedCallBudgets, Stage,
|
||||
client::{CallDeadline, EncodedFrame, read_frame, write_frame},
|
||||
};
|
||||
use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
|
||||
use tokio::sync::{Mutex as AsyncMutex, OwnedSemaphorePermit, Semaphore, mpsc};
|
||||
|
||||
use super::{
|
||||
CallbackHandler, RpcError, RpcLimits, RpcReply, RpcSessionDiagnostic, RpcSessionLifecycle,
|
||||
RpcSessionLoad, Shared, dispatch,
|
||||
};
|
||||
use crate::{PreparedPackage, RpcStream, stream::StreamIngress};
|
||||
|
||||
use super::{admission::ExecutionCapacity, dispatch};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RpcLimits {
|
||||
/// 受管回调与观察队列的缓冲预算,不是请求正文或 IPC 消息总量上限
|
||||
pub maximum_buffered_body_bytes: usize,
|
||||
/// 包含初次排队、下游等待、取消确认和未消费流的总在途上限
|
||||
pub maximum_calls: usize,
|
||||
pub maximum_executing_calls: usize,
|
||||
pub maximum_queued_calls: usize,
|
||||
pub maximum_queue_timeout: Duration,
|
||||
pub maximum_callbacks: usize,
|
||||
pub handshake_timeout: Duration,
|
||||
pub maximum_call_timeout: Duration,
|
||||
pub maximum_stderr_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for RpcLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
maximum_buffered_body_bytes: 1024 * 1024,
|
||||
maximum_calls: 256,
|
||||
maximum_executing_calls: 64,
|
||||
maximum_queued_calls: 64,
|
||||
maximum_queue_timeout: Duration::from_secs(30),
|
||||
maximum_callbacks: 64,
|
||||
handshake_timeout: Duration::from_secs(5),
|
||||
maximum_call_timeout: Duration::from_secs(120),
|
||||
maximum_stderr_bytes: 64 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub enum RpcError {
|
||||
#[error("plugin process could not be started: {0}")]
|
||||
Start(#[source] ProcessStartError),
|
||||
#[error("plugin handshake does not match the prepared instance")]
|
||||
Handshake(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin protocol is invalid")]
|
||||
Protocol(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin returned an invalid response at {0:?}")]
|
||||
InvalidResponse(Stage, #[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin process or transport has stopped")]
|
||||
Closed(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin call exceeded its deadline")]
|
||||
Timeout,
|
||||
#[error("plugin session was cancelled")]
|
||||
Cancelled,
|
||||
#[error("plugin call capacity is exhausted")]
|
||||
Capacity,
|
||||
#[error("plugin admission capacity is exhausted")]
|
||||
Overloaded,
|
||||
#[error("plugin call context is invalid")]
|
||||
Context(#[source] Option<gateway_core::error::ErrorSource>),
|
||||
#[error("plugin returned an error")]
|
||||
Remote(PluginFault),
|
||||
}
|
||||
|
||||
impl RpcError {
|
||||
pub(crate) fn is_overloaded(&self) -> bool {
|
||||
matches!(self, Self::Overloaded)
|
||||
|| matches!(self, Self::Remote(fault) if fault.code == gateway_plugin_sdk::ErrorCode::Capacity)
|
||||
}
|
||||
|
||||
/// 会话状态与请求错误共用安全分类,不展开插件返回正文
|
||||
pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) {
|
||||
match self {
|
||||
RpcError::Start(_) => ("process_start", "插件进程启动失败"),
|
||||
RpcError::Handshake(_) => ("handshake", "插件握手失败"),
|
||||
RpcError::Protocol(_) => ("protocol", "插件协议错误"),
|
||||
RpcError::InvalidResponse(stage, _) => (
|
||||
"invalid_response",
|
||||
match stage {
|
||||
Stage::Registration => "注册阶段:插件返回了无效响应",
|
||||
Stage::Configuration => "配置阶段:插件返回了无效响应",
|
||||
Stage::Authentication => "认证阶段:插件返回了无效响应",
|
||||
Stage::Routing => "路由阶段:插件返回了无效响应",
|
||||
Stage::Scheduling => "调度阶段:插件返回了无效响应",
|
||||
Stage::Retry => "重试阶段:插件返回了无效响应",
|
||||
Stage::Http => "HTTP 中间件:插件返回了无效响应",
|
||||
Stage::Service => "服务中间件:插件返回了无效响应",
|
||||
Stage::WebSocket => "WebSocket 中间件:插件返回了无效响应",
|
||||
Stage::Request => "请求中间件:插件返回了无效响应",
|
||||
Stage::Attempt => "尝试中间件:插件返回了无效响应",
|
||||
Stage::Outbound => "出站准备中间件:插件返回了无效响应",
|
||||
Stage::Upstream => "上游适配阶段:插件返回了无效响应",
|
||||
Stage::Observation => "观察阶段:插件返回了无效响应",
|
||||
Stage::Management | Stage::PublicManagement => "管理阶段:插件返回了无效响应",
|
||||
Stage::CommandLine => "命令行阶段:插件返回了无效响应",
|
||||
Stage::Maintenance => "维护阶段:插件返回了无效响应",
|
||||
},
|
||||
),
|
||||
RpcError::Closed(_) => ("closed", "插件进程或传输已停止"),
|
||||
RpcError::Timeout => ("timeout", "插件调用超时"),
|
||||
RpcError::Cancelled => ("cancelled", "插件会话已取消"),
|
||||
RpcError::Capacity => ("capacity", "插件调用容量已耗尽"),
|
||||
RpcError::Overloaded => ("overloaded", "插件暂时繁忙"),
|
||||
RpcError::Context(_) => ("context", "插件调用上下文无效"),
|
||||
RpcError::Remote(_) => ("remote", "插件返回错误"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum RpcSessionDiagnostic {
|
||||
Ready,
|
||||
Quiescing,
|
||||
Failed {
|
||||
code: &'static str,
|
||||
message: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) struct RpcSessionLifecycle {
|
||||
failure: Arc<dyn Fn(Duration) + Send + Sync>,
|
||||
end: Option<Box<dyn FnOnce() + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl RpcSessionLifecycle {
|
||||
pub(crate) fn new(
|
||||
failure: impl Fn(Duration) + Send + Sync + 'static,
|
||||
end: impl FnOnce() + Send + Sync + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
failure: Arc::new(failure),
|
||||
end: Some(Box::new(end)),
|
||||
}
|
||||
}
|
||||
|
||||
fn fail(&self, uptime: Duration) {
|
||||
(self.failure)(uptime);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RpcSessionLifecycle {
|
||||
fn drop(&mut self) {
|
||||
if let Some(end) = self.end.take() {
|
||||
end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不派生 Debug,防止原始请求、凭据或模型输出进入宿主诊断
|
||||
pub struct RpcReply {
|
||||
pub result: serde_json::Value,
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
/// 具体回调在领域适配器校验账号、出站与执行身份;上下文只取自宿主在途调用表
|
||||
pub trait CallbackHandler: Send + Sync {
|
||||
/// 在 Call 可见之前建立资源范围;不得执行 I/O 或调用 RPC
|
||||
fn begin(&self, _context: &CallContext, _deadline: &CallDeadline) {}
|
||||
|
||||
/// 调用结束或取消时立即撤销其句柄;实现必须幂等且不能阻塞
|
||||
fn finish(&self, _context: &CallContext) {}
|
||||
|
||||
/// 首个流式结果已到达;资源仍绑定原调用并在终态统一回收
|
||||
fn streaming(&self, _context: &CallContext) {}
|
||||
|
||||
fn call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
method: String,
|
||||
params: serde_json::Value,
|
||||
payload: Vec<u8>,
|
||||
) -> BoxFuture<'static, Result<RpcReply, PluginFault>>;
|
||||
}
|
||||
|
||||
pub struct RpcSession {
|
||||
shared: Arc<Shared>,
|
||||
handshake: Handshake,
|
||||
budgets: ResolvedCallBudgets,
|
||||
data: mpsc::Sender<EncodedFrame>,
|
||||
slots: Arc<Semaphore>,
|
||||
next_call: AsyncMutex<u64>,
|
||||
@@ -211,365 +43,19 @@ impl RpcSession {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Shared {
|
||||
state: Mutex<State>,
|
||||
started_at: Instant,
|
||||
expected_stop: AtomicBool,
|
||||
failure_observer: Option<Arc<dyn Fn(Duration) + Send + Sync>>,
|
||||
pub control: mpsc::Sender<EncodedFrame>,
|
||||
pub process: ProcessControl,
|
||||
pub stopped: watch::Sender<bool>,
|
||||
callback_slots: Arc<Semaphore>,
|
||||
next_slots: Arc<Semaphore>,
|
||||
execution: ExecutionCapacity,
|
||||
}
|
||||
|
||||
struct State {
|
||||
failure: Option<RpcError>,
|
||||
calls: BTreeMap<u64, PendingCall>,
|
||||
}
|
||||
|
||||
struct PendingCall {
|
||||
_slot: Arc<OwnedSemaphorePermit>,
|
||||
execution: Option<OwnedSemaphorePermit>,
|
||||
deadline: CallDeadline,
|
||||
handler: Arc<dyn CallbackHandler>,
|
||||
transmitted: bool,
|
||||
cancellation: Option<RpcError>,
|
||||
context: CallContext,
|
||||
result: Option<oneshot::Sender<Result<RpcReply, RpcError>>>,
|
||||
stream: Option<StreamIngress>,
|
||||
callbacks: Vec<tokio::task::AbortHandle>,
|
||||
}
|
||||
|
||||
impl PendingCall {
|
||||
fn finish(mut self, result: Result<RpcReply, RpcError>) {
|
||||
for callback in self.callbacks.drain(..) {
|
||||
callback.abort();
|
||||
}
|
||||
let stream = self.stream.take();
|
||||
let sender = self.result.take();
|
||||
// 先同步关闭调用资源,Core 才能在收到结果时看到全部发送副作用
|
||||
drop(self);
|
||||
if let Some(stream) = stream {
|
||||
stream.finish(result.as_ref().map(|_| ()).map_err(Clone::clone));
|
||||
}
|
||||
if let Some(sender) = sender {
|
||||
let _ = sender.send(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingCall {
|
||||
fn drop(&mut self) {
|
||||
self.handler.finish(&self.context);
|
||||
}
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
fn state(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub fn fail(&self, error: RpcError) {
|
||||
let uptime = self.started_at.elapsed();
|
||||
let (calls, failure_observer) = {
|
||||
let mut state = self.state();
|
||||
if state.failure.is_some() {
|
||||
return;
|
||||
}
|
||||
state.failure = Some(error.clone());
|
||||
let observer = (!self.expected_stop.load(Ordering::Acquire))
|
||||
.then(|| self.failure_observer.clone())
|
||||
.flatten();
|
||||
(std::mem::take(&mut state.calls), observer)
|
||||
};
|
||||
self.execution.close();
|
||||
for call in calls.into_values() {
|
||||
call.finish(Err(error.clone()));
|
||||
}
|
||||
if let Some(observer) = failure_observer {
|
||||
observer(uptime);
|
||||
}
|
||||
self.stopped.send_replace(true);
|
||||
self.process.stop();
|
||||
}
|
||||
|
||||
fn expect_stop(&self) {
|
||||
self.expected_stop.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn finish(&self, id: u64, result: Result<RpcReply, RpcError>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if matches!(&result, Err(RpcError::Remote(fault)) if fault.code == gateway_plugin_sdk::ErrorCode::Timeout)
|
||||
{
|
||||
log_timeout(&call.context, "plugin", "initial_reply", "remote_timeout");
|
||||
}
|
||||
if call.stream.is_some() && result.is_ok() {
|
||||
if call.context.resource_stream {
|
||||
call.deadline.clear();
|
||||
}
|
||||
let initial = call.result.take().ok_or(RpcError::Protocol(None))?;
|
||||
call.execution.take();
|
||||
call.handler.streaming(&call.context);
|
||||
let _ = initial.send(result);
|
||||
} else {
|
||||
let pending = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
|
||||
drop(state);
|
||||
pending.finish(result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stream_chunk(&self, id: u64, sequence: u64, payload: Vec<u8>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
|
||||
if call.cancellation.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
if call.result.is_some() {
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
call.stream
|
||||
.as_mut()
|
||||
.ok_or(RpcError::Protocol(None))?
|
||||
.receive(sequence, payload)
|
||||
}
|
||||
|
||||
pub fn stream_end(&self, id: u64, error: Option<PluginFault>) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
if state
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let call = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
|
||||
drop(state);
|
||||
if call.stream.is_none() || call.result.is_some() {
|
||||
call.finish(Err(RpcError::Protocol(None)));
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
let result = error.map_or_else(
|
||||
|| {
|
||||
Ok(RpcReply {
|
||||
result: serde_json::Value::Null,
|
||||
payload: Vec::new(),
|
||||
})
|
||||
},
|
||||
|error| Err(RpcError::Remote(error)),
|
||||
);
|
||||
call.finish(result);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn release_stream_credit(&self, id: u64, bytes: u32) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
// 完成后仍可消费已排队数据,无须再给已关闭的流发信用
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return Ok(());
|
||||
};
|
||||
call.stream
|
||||
.as_mut()
|
||||
.ok_or(RpcError::Protocol(None))?
|
||||
.release(bytes)?;
|
||||
drop(state);
|
||||
self.send_control(Frame::control(Message::Credit {
|
||||
id,
|
||||
bytes,
|
||||
frames: 1,
|
||||
}));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn context(&self, id: u64) -> Option<CallContext> {
|
||||
self.state()
|
||||
.calls
|
||||
.get(&id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
.map(|call| call.context.clone())
|
||||
}
|
||||
|
||||
pub fn pause_next(&self, id: u64) -> Option<CallDeadlinePause> {
|
||||
let mut state = self.state();
|
||||
let call = state
|
||||
.calls
|
||||
.get_mut(&id)
|
||||
.filter(|call| call.cancellation.is_none())?;
|
||||
let pause = call.deadline.pause()?;
|
||||
call.execution.take();
|
||||
Some(pause)
|
||||
}
|
||||
|
||||
pub async fn resume_next(&self, id: u64) -> Result<(), RpcError> {
|
||||
{
|
||||
let state = self.state();
|
||||
let call = state
|
||||
.calls
|
||||
.get(&id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
.ok_or(RpcError::Cancelled)?;
|
||||
// 已交付响应头的流只占在途资源,不重新占用首结果执行名额
|
||||
if call.result.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let permit = self.execution.resume().await?;
|
||||
let mut state = self.state();
|
||||
let call = state
|
||||
.calls
|
||||
.get_mut(&id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
.ok_or(RpcError::Cancelled)?;
|
||||
if call.result.is_some() {
|
||||
call.execution = Some(permit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_cancelling(&self, id: u64) -> bool {
|
||||
self.state()
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
}
|
||||
|
||||
pub fn mark_transmitted(&self, id: u64) -> bool {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return false;
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return false;
|
||||
}
|
||||
call.transmitted = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// 先撤销调用权限,再等待对端确认;只有无响应对端才终止整个 incarnation
|
||||
pub fn cancel(self: &Arc<Self>, id: u64, error: RpcError) {
|
||||
let mut state = self.state();
|
||||
let Some(call) = state.calls.get_mut(&id) else {
|
||||
return;
|
||||
};
|
||||
if call.cancellation.is_some() {
|
||||
return;
|
||||
}
|
||||
if matches!(error, RpcError::Timeout) {
|
||||
log_timeout(
|
||||
&call.context,
|
||||
"runtime",
|
||||
if call.result.is_some() {
|
||||
"initial_reply"
|
||||
} else {
|
||||
"response_stream"
|
||||
},
|
||||
"processing_budget_exhausted",
|
||||
);
|
||||
}
|
||||
if !call.transmitted {
|
||||
if let Some(call) = state.calls.remove(&id) {
|
||||
call.finish(Err(error));
|
||||
}
|
||||
return;
|
||||
}
|
||||
call.cancellation = Some(error.clone());
|
||||
call.handler.finish(&call.context);
|
||||
for callback in call.callbacks.drain(..) {
|
||||
callback.abort();
|
||||
}
|
||||
if let Some(result) = call.result.take() {
|
||||
let _ = result.send(Err(error.clone()));
|
||||
}
|
||||
if let Some(stream) = call.stream.take() {
|
||||
stream.finish(Err(error.clone()));
|
||||
}
|
||||
drop(state);
|
||||
self.send_control(Frame::control(Message::Cancel { id }));
|
||||
let shared = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
if shared.is_cancelling(id) {
|
||||
shared.fail(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acknowledge_cancellation(&self, id: u64) -> Result<(), RpcError> {
|
||||
let mut state = self.state();
|
||||
if !state
|
||||
.calls
|
||||
.get(&id)
|
||||
.is_some_and(|call| call.cancellation.is_some())
|
||||
{
|
||||
return Err(RpcError::Protocol(None));
|
||||
}
|
||||
state.calls.remove(&id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn track_callback(&self, parent_id: u64, callback: tokio::task::AbortHandle) -> bool {
|
||||
if let Some(call) = self
|
||||
.state()
|
||||
.calls
|
||||
.get_mut(&parent_id)
|
||||
.filter(|call| call.cancellation.is_none())
|
||||
{
|
||||
// 已完成回调的句柄不继续占用请求预算
|
||||
call.callbacks.retain(|handle| !handle.is_finished());
|
||||
call.callbacks.push(callback);
|
||||
true
|
||||
} else {
|
||||
callback.abort();
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send_control(&self, frame: Frame) {
|
||||
let Ok(frame) = EncodedFrame::new(frame) else {
|
||||
self.fail(RpcError::Protocol(None));
|
||||
return;
|
||||
};
|
||||
self.send_encoded_control(frame);
|
||||
}
|
||||
|
||||
pub(super) fn send_encoded_control(&self, frame: EncodedFrame) {
|
||||
if self.control.try_send(frame).is_err() {
|
||||
self.fail(RpcError::Capacity);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_callback_slot(&self, method: &str) -> Option<OwnedSemaphorePermit> {
|
||||
let slots = if method == gateway_plugin_sdk::call::middleware::NEXT_METHOD {
|
||||
&self.next_slots
|
||||
} else {
|
||||
&self.callback_slots
|
||||
};
|
||||
slots.clone().try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
async fn wait_for_callbacks(&self, limits: RpcLimits) {
|
||||
for (slots, maximum) in [
|
||||
(&self.callback_slots, limits.maximum_callbacks),
|
||||
(&self.next_slots, limits.maximum_calls),
|
||||
] {
|
||||
if let Ok(permits) = slots.clone().acquire_many_owned(maximum as u32).await {
|
||||
drop(permits);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RpcSession {
|
||||
pub(crate) const fn maximum_call_timeout(&self) -> Duration {
|
||||
self.limits.maximum_call_timeout
|
||||
pub fn budget(&self, stage: Stage) -> Duration {
|
||||
self.budgets.budget(stage)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn budget_source(&self, stage: Stage) -> CallBudgetSource {
|
||||
self.budgets.source(stage)
|
||||
}
|
||||
|
||||
/// 替换维护任务前确认旧调用已取消或所属进程退出
|
||||
pub(crate) async fn settle_cancellations(&self) {
|
||||
self.shared.settle_cancellations().await;
|
||||
}
|
||||
|
||||
pub async fn start(
|
||||
@@ -616,11 +102,10 @@ impl RpcSession {
|
||||
|| limits.maximum_queued_calls > 256
|
||||
|| limits.maximum_queue_timeout.is_zero()
|
||||
|| limits.maximum_callbacks == 0
|
||||
|| limits.maximum_callbacks > 256
|
||||
|| limits.maximum_callbacks > 512
|
||||
|| limits.maximum_buffered_body_bytes < 1024
|
||||
|| limits.maximum_buffered_body_bytes > 16 * 1024 * 1024
|
||||
|| limits.handshake_timeout.is_zero()
|
||||
|| limits.maximum_call_timeout.is_zero()
|
||||
{
|
||||
return Err(RpcError::Context(None));
|
||||
}
|
||||
@@ -691,11 +176,12 @@ impl RpcSession {
|
||||
Message::Ready {
|
||||
protocol_version,
|
||||
incarnation,
|
||||
call_budgets,
|
||||
} if protocol_version == handshake.protocol_version
|
||||
&& incarnation == handshake.incarnation
|
||||
&& reply.payload.is_empty() =>
|
||||
{
|
||||
Ok(())
|
||||
ResolvedCallBudgets::resolve(&call_budgets).map_err(RpcError::Budget)
|
||||
}
|
||||
_ => Err(RpcError::Handshake(None)),
|
||||
}
|
||||
@@ -703,41 +189,29 @@ impl RpcSession {
|
||||
.await
|
||||
.map_err(|_| RpcError::Timeout)
|
||||
.and_then(|result| result);
|
||||
if let Err(error) = ready {
|
||||
if let Some(lifecycle) = &lifecycle {
|
||||
// 只有 Ready 之后的存活才可复位预算;长时间卡在握手不等于稳定运行
|
||||
lifecycle.fail(Duration::ZERO);
|
||||
let budgets = match ready {
|
||||
Ok(budgets) => budgets,
|
||||
Err(error) => {
|
||||
if let Some(lifecycle) = &lifecycle {
|
||||
// 只有 Ready 之后的存活才可复位预算;长时间卡在握手不等于稳定运行
|
||||
lifecycle.fail(Duration::ZERO);
|
||||
}
|
||||
process.stop();
|
||||
process.exited().await;
|
||||
return Err(error);
|
||||
}
|
||||
process.stop();
|
||||
process.exited().await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let (data, data_receiver) = mpsc::channel(limits.maximum_calls);
|
||||
let (control, control_receiver) =
|
||||
mpsc::channel(limits.maximum_callbacks + 2 * limits.maximum_calls + 8);
|
||||
let (stopped, _) = watch::channel(false);
|
||||
let callback_slots = Arc::new(Semaphore::new(limits.maximum_callbacks));
|
||||
let shared = Arc::new(Shared {
|
||||
state: Mutex::new(State {
|
||||
failure: None,
|
||||
calls: BTreeMap::new(),
|
||||
}),
|
||||
started_at: Instant::now(),
|
||||
expected_stop: AtomicBool::new(false),
|
||||
failure_observer: lifecycle
|
||||
let shared = Arc::new(Shared::new(
|
||||
limits,
|
||||
process,
|
||||
control,
|
||||
lifecycle
|
||||
.as_ref()
|
||||
.map(|lifecycle| Arc::clone(&lifecycle.failure)),
|
||||
control,
|
||||
process,
|
||||
stopped,
|
||||
callback_slots,
|
||||
next_slots: Arc::new(Semaphore::new(limits.maximum_calls)),
|
||||
execution: ExecutionCapacity::new(
|
||||
limits.maximum_executing_calls,
|
||||
limits.maximum_queued_calls,
|
||||
limits.maximum_queue_timeout,
|
||||
),
|
||||
});
|
||||
));
|
||||
dispatch::start(
|
||||
input,
|
||||
output,
|
||||
@@ -760,6 +234,7 @@ impl RpcSession {
|
||||
Ok(Self {
|
||||
shared,
|
||||
handshake,
|
||||
budgets,
|
||||
data,
|
||||
slots: Arc::new(Semaphore::new(limits.maximum_calls)),
|
||||
next_call: AsyncMutex::new(1),
|
||||
@@ -771,23 +246,17 @@ impl RpcSession {
|
||||
|
||||
#[must_use]
|
||||
pub fn is_ready(&self) -> bool {
|
||||
self.shared.failure_diagnostic().is_none()
|
||||
}
|
||||
|
||||
pub(crate) fn load(&self) -> RpcSessionLoad {
|
||||
self.shared
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.failure
|
||||
.is_none()
|
||||
.load(self.limits, self.slots.available_permits())
|
||||
}
|
||||
|
||||
/// 仅返回稳定分类与固定文案,绝不展开插件 fault、stderr 或请求内容
|
||||
pub(crate) fn diagnostic(&self) -> RpcSessionDiagnostic {
|
||||
let state = self
|
||||
.shared
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(error) = &state.failure {
|
||||
let (code, message) = error.diagnostic();
|
||||
if let Some((code, message)) = self.shared.failure_diagnostic() {
|
||||
return RpcSessionDiagnostic::Failed { code, message };
|
||||
}
|
||||
if self.slots.is_closed() {
|
||||
@@ -799,8 +268,8 @@ impl RpcSession {
|
||||
|
||||
#[must_use]
|
||||
pub fn context(&self, stage: gateway_plugin_sdk::Stage, timeout: Duration) -> CallContext {
|
||||
// 宿主生成的上下文受本会话预算约束;固定注册期限不能使较小的总预算无法启动
|
||||
let timeout = timeout.min(self.limits.maximum_call_timeout);
|
||||
// 调用方可以传入更短的外部剩余期限,不能扩大实例已协商的阶段预算
|
||||
let timeout = timeout.min(self.budget(stage));
|
||||
CallContext {
|
||||
request: None,
|
||||
call_id: 0,
|
||||
@@ -823,7 +292,7 @@ impl RpcSession {
|
||||
|| context.generation != self.handshake.generation
|
||||
|| context.incarnation != self.handshake.incarnation
|
||||
|| context.timeout_ms == 0
|
||||
|| Duration::from_millis(context.timeout_ms) > self.limits.maximum_call_timeout
|
||||
|| Duration::from_millis(context.timeout_ms) > self.budget(context.stage)
|
||||
|| method.is_empty()
|
||||
|| method.len() > 128
|
||||
{
|
||||
@@ -832,44 +301,6 @@ impl RpcSession {
|
||||
Ok(Duration::from_millis(context.timeout_ms))
|
||||
}
|
||||
|
||||
fn begin_call(
|
||||
&self,
|
||||
context: CallContext,
|
||||
deadline: tokio::time::Instant,
|
||||
stream: Option<StreamIngress>,
|
||||
slot: Arc<OwnedSemaphorePermit>,
|
||||
execution: OwnedSemaphorePermit,
|
||||
) -> Result<StartedCall, RpcError> {
|
||||
let id = context.call_id;
|
||||
let (result, received) = oneshot::channel();
|
||||
let mut state = self.shared.state();
|
||||
if let Some(error) = &state.failure {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let deadline = CallDeadline::new(deadline);
|
||||
self.callbacks.begin(&context, &deadline);
|
||||
state.calls.insert(
|
||||
id,
|
||||
PendingCall {
|
||||
_slot: slot,
|
||||
execution: Some(execution),
|
||||
deadline: deadline.clone(),
|
||||
handler: self.callbacks.clone(),
|
||||
transmitted: false,
|
||||
cancellation: None,
|
||||
context,
|
||||
result: Some(result),
|
||||
stream,
|
||||
callbacks: Vec::new(),
|
||||
},
|
||||
);
|
||||
Ok(StartedCall {
|
||||
id,
|
||||
deadline,
|
||||
received,
|
||||
})
|
||||
}
|
||||
|
||||
async fn send_call(
|
||||
&self,
|
||||
method: &str,
|
||||
@@ -887,7 +318,19 @@ impl RpcSession {
|
||||
RpcError::Overloaded
|
||||
}
|
||||
})?);
|
||||
let execution = self.shared.execution.enter().await?;
|
||||
let queued_at = Instant::now();
|
||||
let execution = self.shared.execution.enter().await;
|
||||
tracing::debug!(
|
||||
event = "plugin.rpc.admission",
|
||||
instance_id = context.instance_id,
|
||||
stage = ?context.stage,
|
||||
request_id = context.request_id,
|
||||
attempt_id = context.attempt_id,
|
||||
wait_ms = queued_at.elapsed().as_millis() as u64,
|
||||
outcome = execution.as_ref().map_or_else(|error| error.diagnostic().0, |_| "admitted"),
|
||||
"插件 RPC 执行准入完成"
|
||||
);
|
||||
let execution = execution?;
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
// FIFO 锁同时拥有 ID 与 data 入队顺序;Call/初始 Credit 入队后必须立即释放,
|
||||
// 不能把插件回复或重入回调纳入串行区,也不能为等待锁重置调用期限
|
||||
@@ -911,7 +354,14 @@ impl RpcSession {
|
||||
let frame = EncodedFrame::new(frame).map_err(|source| {
|
||||
RpcError::Context(Some(gateway_core::error::ErrorSource::new(source)))
|
||||
})?;
|
||||
let started = self.begin_call(context, deadline, stream, slot.clone(), execution)?;
|
||||
let started = self.shared.begin_call(
|
||||
context,
|
||||
deadline,
|
||||
stream,
|
||||
slot.clone(),
|
||||
execution,
|
||||
self.callbacks.clone(),
|
||||
)?;
|
||||
let mut guard = CallGuard {
|
||||
shared: Arc::clone(&self.shared),
|
||||
id: started.id,
|
||||
@@ -1085,36 +535,9 @@ impl Drop for CallGuard {
|
||||
}
|
||||
}
|
||||
|
||||
struct StartedCall {
|
||||
id: u64,
|
||||
deadline: CallDeadline,
|
||||
received: oneshot::Receiver<Result<RpcReply, RpcError>>,
|
||||
}
|
||||
|
||||
struct CompletedCall {
|
||||
slot: Arc<OwnedSemaphorePermit>,
|
||||
id: u64,
|
||||
deadline: CallDeadline,
|
||||
reply: RpcReply,
|
||||
}
|
||||
|
||||
fn log_timeout(
|
||||
context: &CallContext,
|
||||
owner: &'static str,
|
||||
phase: &'static str,
|
||||
reason: &'static str,
|
||||
) {
|
||||
tracing::warn!(
|
||||
event = "plugin.rpc.timeout",
|
||||
owner,
|
||||
phase,
|
||||
instance_id = context.instance_id,
|
||||
call_id = context.call_id,
|
||||
request_id = context.request_id,
|
||||
attempt_id = context.attempt_id,
|
||||
stage = ?context.stage,
|
||||
budget_ms = context.timeout_ms,
|
||||
reason,
|
||||
"插件 RPC 调用超时"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,120 @@ use serde_json::json;
|
||||
|
||||
use crate::support::environment::Environment;
|
||||
|
||||
#[derive(Default)]
|
||||
struct BudgetAccountAccess {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
first_started: tokio::sync::Notify,
|
||||
second_started: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl gateway_admin::ports::plugin_accounts::PluginAccountAccess for BudgetAccountAccess {
|
||||
async fn list(
|
||||
&self,
|
||||
_: gateway_admin::model::provider_credentials::PluginAccountListQuery,
|
||||
) -> Result<
|
||||
gateway_admin::model::provider_credentials::PluginAccountPage,
|
||||
gateway_admin::model::AdminError,
|
||||
> {
|
||||
panic!("账号提交不应读取账号列表")
|
||||
}
|
||||
|
||||
async fn get_runtime(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
) -> Result<gateway_admin::model::accounts::AccountRecord, gateway_admin::model::AdminError>
|
||||
{
|
||||
panic!("创建账号不应读取已有账号")
|
||||
}
|
||||
|
||||
async fn get_credential(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
) -> Result<
|
||||
gateway_admin::model::provider_credentials::PluginAccountCredential,
|
||||
gateway_admin::model::AdminError,
|
||||
> {
|
||||
panic!("账号提交不应读取凭据")
|
||||
}
|
||||
|
||||
async fn save(
|
||||
&self,
|
||||
_: gateway_admin::model::provider_credentials::PreparedPluginAccountSave,
|
||||
_: &gateway_admin::model::MutationContext,
|
||||
) -> Result<
|
||||
gateway_admin::model::provider_credentials::PluginAccountSaveResult,
|
||||
gateway_admin::model::AdminError,
|
||||
> {
|
||||
if self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
|
||||
self.first_started.notify_one();
|
||||
tokio::time::sleep(std::time::Duration::from_secs(70)).await;
|
||||
Ok(
|
||||
gateway_admin::model::provider_credentials::PluginAccountSaveResult {
|
||||
config_revision: gateway_admin::model::Revision::new(2).unwrap(),
|
||||
account_id: gateway_core::account::ProviderAccountId::new(
|
||||
"acct_command_budget",
|
||||
)
|
||||
.unwrap(),
|
||||
credential_revision: gateway_admin::model::Revision::new(1).unwrap(),
|
||||
},
|
||||
)
|
||||
} else {
|
||||
self.second_started.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_commits_share_the_commands_declared_deadline_and_report_partial_completion() {
|
||||
use gateway_admin::ports::plugin_accounts::PluginAccountAccess;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
let account = json!({"action":"create","provider_id":"openai","facts":{
|
||||
"name":"预算测试","authentication_kind":"api_key","material":{"key":"fixture-only"}
|
||||
}});
|
||||
let config = json!({
|
||||
"call_budgets_ms":{"command_line":180_000},
|
||||
"command_registration":{"commands":[{"name":"save","description":"提交账号"}]},
|
||||
"command_result":{"stdout":"","stderr":"","exit_code":0,"accounts":[account, account]},
|
||||
});
|
||||
let (_directory, _store, runtime) = super::setup(config).await;
|
||||
let probe = Arc::new(BudgetAccountAccess::default());
|
||||
let accounts: Arc<dyn PluginAccountAccess> = probe.clone();
|
||||
runtime.bind_account_ports(&accounts).unwrap();
|
||||
let commands = runtime.prepare_command_line().await.unwrap();
|
||||
let task = tokio::spawn(async move {
|
||||
let result = commands.execute("commands", "save", &[]).await;
|
||||
(commands, result)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), probe.first_started.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(121)).await;
|
||||
probe.second_started.notified().await;
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"账号提交沿用声明预算,不能被默认期限截断"
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
let (commands, result) = task.await.unwrap();
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(PluginCommandError::Incomplete {
|
||||
saved_accounts: 1,
|
||||
reason: "超过命令总期限",
|
||||
})
|
||||
),
|
||||
"后续账号提交不能重新获得完整预算"
|
||||
);
|
||||
commands.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_requests_have_a_body_limit_independent_of_the_response_buffer() {
|
||||
use gateway_plugin_sdk::call::model::request_payload_limit;
|
||||
|
||||
@@ -174,8 +174,7 @@ async fn failed_or_cancelled_commands_are_not_replayed_and_children_are_reaped()
|
||||
} else {
|
||||
// 正常完成注册并确认命令已进入子进程后,只推进执行期限
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(limits.maximum_call_timeout + Duration::from_millis(1))
|
||||
.await;
|
||||
tokio::time::advance(Duration::from_secs(120) + Duration::from_millis(1)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -209,3 +208,46 @@ async fn failed_or_cancelled_commands_are_not_replayed_and_children_are_reaped()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn command_declared_budget_survives_the_default_and_expires_at_its_own_deadline() {
|
||||
use std::time::Duration;
|
||||
|
||||
let mut config = configuration(json!([]));
|
||||
config["command_wait"] = json!(true);
|
||||
config["call_budgets_ms"] = json!({"command_line":180_000});
|
||||
let (directory, store, runtime) = setup(config).await;
|
||||
let marker = directory.path().join("command-budget.jsonl");
|
||||
store.snapshot.lock().unwrap().instances[0].configuration["command_marker"] = json!(marker);
|
||||
let commands = runtime.prepare_command_line().await.unwrap();
|
||||
let task = tokio::spawn(async move {
|
||||
let result = commands.execute("commands", "inspect", &[]).await;
|
||||
(commands, result)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !marker.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(121)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished(), "声明的命令预算不受默认 120 秒截断");
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
let (commands, result) = task.await.unwrap();
|
||||
tokio::time::resume();
|
||||
let error = result.err().unwrap();
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_plugin_runtime::PluginCommandError::Incomplete {
|
||||
saved_accounts: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert!(error.to_string().contains("超时") || error.to_string().contains("总期限"));
|
||||
commands.shutdown().await;
|
||||
assert_eq!(std::fs::read_to_string(marker).unwrap().lines().count(), 1);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
//! 验证回调方法精确分派,未知方法不进入其他能力的授权或资源处理
|
||||
|
||||
use gateway_admin::{
|
||||
model::plugins::management::PluginManagementRequest, ports::plugin_management::PluginManagement,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_callback_methods_are_unsupported_without_affecting_known_methods() {
|
||||
let Some(environment) = crate::support::environment::Environment::create().await else {
|
||||
eprintln!("SKIP: plugin integration environment absent");
|
||||
return;
|
||||
};
|
||||
let (runtime, core) = environment.plugin(json!({
|
||||
"management_registration":{"routes":[{"method":"GET","path":"dispatch","request_content_types":[],"response_content_types":["application/json"]}]},
|
||||
"data_queries":[
|
||||
{"method":"host.unknown","query":{}},
|
||||
{"method":"host.middleware.unknown","query":{}},
|
||||
{"method":"host.upstream.unknown","query":{}},
|
||||
{"method":"host.http.unknown","query":{}},
|
||||
{"method":gateway_plugin_sdk::call::host::HTTP_DO,"query":{}},
|
||||
{"method":gateway_plugin_sdk::call::middleware::NEXT_METHOD,"query":{}}
|
||||
]
|
||||
})).await;
|
||||
let generation = core
|
||||
.snapshots()
|
||||
.acquire()
|
||||
.unwrap()
|
||||
.extensions()
|
||||
.unwrap()
|
||||
.clone();
|
||||
let view = runtime.views(&generation).await.unwrap().remove(0);
|
||||
let response = runtime
|
||||
.handle(
|
||||
&generation,
|
||||
&view.target,
|
||||
PluginManagementRequest {
|
||||
request_context: None,
|
||||
headers: Vec::new(),
|
||||
method: "GET".into(),
|
||||
path: "dispatch".into(),
|
||||
query: String::new(),
|
||||
content_type: None,
|
||||
body: Vec::new(),
|
||||
request_id: "callback-dispatch".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let results: Vec<Value> = serde_json::from_slice(&response.body).unwrap();
|
||||
let codes: Vec<_> = results
|
||||
.iter()
|
||||
.map(|result| result["error"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
codes,
|
||||
[
|
||||
"unsupported",
|
||||
"unsupported",
|
||||
"unsupported",
|
||||
"unsupported",
|
||||
"invalid_input",
|
||||
"permission_denied"
|
||||
]
|
||||
);
|
||||
drop(generation);
|
||||
environment.release_plugin_accounts(&runtime);
|
||||
runtime.shutdown().await;
|
||||
drop(core);
|
||||
drop(runtime);
|
||||
environment.close().await;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 插件宿主回调的测试入口
|
||||
|
||||
mod dispatch;
|
||||
mod key_facts;
|
||||
mod keys;
|
||||
mod log;
|
||||
|
||||
@@ -273,3 +273,117 @@ async fn startup_failure_of_an_unrelated_plugin_preserves_native_authentication_
|
||||
}
|
||||
close(environment, runtime, core).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_verification_reenters_the_authentication_instance_without_model_binding() {
|
||||
assert_callback_identity_queries(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_queries_do_not_require_model_execution_ports() {
|
||||
assert_callback_identity_queries(false).await;
|
||||
}
|
||||
|
||||
async fn assert_callback_identity_queries(include_models: bool) {
|
||||
use gateway_admin::{
|
||||
model::plugins::management::PluginManagementRequest,
|
||||
ports::plugin_management::PluginManagement,
|
||||
};
|
||||
use gateway_plugin_runtime::RpcLimits;
|
||||
let Some(environment) = Environment::create().await else {
|
||||
return;
|
||||
};
|
||||
environment
|
||||
.client_key("key_frontend", "sk-native-fixture")
|
||||
.await;
|
||||
let account = environment.account(None).await;
|
||||
let marker = environment
|
||||
.directory
|
||||
.path()
|
||||
.join("authenticated-facts.json");
|
||||
let mut facts = vec![json!(
|
||||
{"method":"host.data.access.check", "query":{"client_key_id":"key_frontend", "account_ids":[account.as_str(), "acct_missing"]}}
|
||||
)];
|
||||
if include_models {
|
||||
facts.push(json!(
|
||||
{"method":"host.models.list", "params":{"client_key_id":"key_frontend", "protocol":"codex", "client_version":"0.154.0"}}
|
||||
));
|
||||
}
|
||||
let mut config = configuration(
|
||||
json!({"outcome":"authenticated","principal":"external-user"}),
|
||||
true,
|
||||
"External controlled-fixture",
|
||||
);
|
||||
config["sdk_session"] = json!(true);
|
||||
config["frontend_authentication_marker"] = json!(marker);
|
||||
config["auth_queries"] = json!(facts);
|
||||
config["management_registration"] = json!({"routes":[{"method":"GET","path":"facts","request_content_types":[],"response_content_types":["application/json"]}]});
|
||||
let mut queries = vec![json!(
|
||||
{"method":"host.client.verify", "query":{"authorization":"External controlled-fixture"}}
|
||||
)];
|
||||
queries.extend(facts);
|
||||
config["data_queries"] = json!(queries);
|
||||
environment.install_plugin(config).await;
|
||||
let (runtime, core) = environment
|
||||
.runtime_with_identity_port(
|
||||
RpcLimits {
|
||||
maximum_executing_calls: 1,
|
||||
maximum_queue_timeout: std::time::Duration::from_millis(500),
|
||||
..RpcLimits::default()
|
||||
},
|
||||
crate::support::native::provider_registry(),
|
||||
)
|
||||
.await;
|
||||
if include_models {
|
||||
runtime
|
||||
.bind_model_ports(
|
||||
&core.nested_model_execution_port(),
|
||||
&core.affinity_lookup_port(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let snapshot = core.snapshots().acquire().unwrap();
|
||||
let generation = snapshot.extensions().unwrap();
|
||||
let view = runtime.views(generation).await.unwrap().remove(0);
|
||||
let response = runtime
|
||||
.handle(
|
||||
generation,
|
||||
&view.target,
|
||||
PluginManagementRequest {
|
||||
request_context: None,
|
||||
headers: Vec::new(),
|
||||
method: "GET".into(),
|
||||
path: "facts".into(),
|
||||
query: String::new(),
|
||||
content_type: None,
|
||||
body: Vec::new(),
|
||||
request_id: "verify-composition".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let results: Value = serde_json::from_slice(&response.body).unwrap();
|
||||
assert_eq!(results[0], json!({"client_key_id":"key_frontend"}));
|
||||
let authenticated_facts: Value =
|
||||
serde_json::from_slice(&std::fs::read(marker).unwrap()).unwrap();
|
||||
assert_eq!(authenticated_facts[0], results[1]);
|
||||
assert_eq!(
|
||||
authenticated_facts[0]["accounts"],
|
||||
json!([
|
||||
{"account_id":account.as_str(),"allowed":true}, {"account_id":"acct_missing","allowed":false}
|
||||
])
|
||||
);
|
||||
if include_models {
|
||||
assert_eq!(authenticated_facts[1], results[2]);
|
||||
// 此替身只登记执行能力,没有原生目录;查询应成功返回空目录而不是执行绑定错误
|
||||
assert_eq!(results[2], json!({"models":[]}));
|
||||
}
|
||||
assert!(
|
||||
environment
|
||||
.bound_model_requests("key_frontend")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
drop(snapshot);
|
||||
close(environment, runtime, core).await;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,12 @@ use gateway_plugin_sdk::{Capability, Contributions, Stage};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Facts(AtomicUsize, AtomicUsize, Option<String>);
|
||||
struct Facts {
|
||||
reads: AtomicUsize,
|
||||
refreshes: AtomicUsize,
|
||||
email: Option<String>,
|
||||
runtime_account: Option<AccountRecord>,
|
||||
}
|
||||
|
||||
fn account() -> AccountRecord {
|
||||
let at = "2026-01-01T00:00:00Z".parse().unwrap();
|
||||
@@ -71,7 +76,7 @@ impl PluginAccountAccess for Facts {
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<ProviderQuota, AdminError> {
|
||||
self.1.fetch_add(1, Ordering::SeqCst);
|
||||
self.refreshes.fetch_add(1, Ordering::SeqCst);
|
||||
if account.as_str() == "acct_missing" {
|
||||
return Err(AdminError::not_found("missing account"));
|
||||
}
|
||||
@@ -88,19 +93,26 @@ impl PluginAccountAccess for Facts {
|
||||
}
|
||||
|
||||
async fn list(&self, query: PluginAccountListQuery) -> Result<PluginAccountPage, AdminError> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
self.reads.fetch_add(1, Ordering::SeqCst);
|
||||
assert_eq!(query.limit.get(), 1);
|
||||
assert_eq!(query.provider_kind.unwrap().as_str(), "openai");
|
||||
assert_eq!(query.cursor.unwrap().as_str(), "acct_before");
|
||||
let mut account = account();
|
||||
account.email.clone_from(&self.2);
|
||||
account.email.clone_from(&self.email);
|
||||
Ok(PluginAccountPage {
|
||||
accounts: vec![account],
|
||||
next_cursor: Some(ProviderAccountId::new("acct_facts").unwrap()),
|
||||
})
|
||||
}
|
||||
async fn get_runtime(&self, _: &ProviderAccountId) -> Result<AccountRecord, AdminError> {
|
||||
panic!("事实回调只使用对应的窄查询")
|
||||
async fn get_runtime(&self, id: &ProviderAccountId) -> Result<AccountRecord, AdminError> {
|
||||
let account = self
|
||||
.runtime_account
|
||||
.as_ref()
|
||||
.expect("事实回调只使用对应的窄查询");
|
||||
if account.id != id.as_str() {
|
||||
return Err(AdminError::not_found("fixture account is unavailable"));
|
||||
}
|
||||
Ok(account.clone())
|
||||
}
|
||||
async fn get_credential(
|
||||
&self,
|
||||
@@ -116,10 +128,14 @@ impl PluginAccountAccess for Facts {
|
||||
panic!("不允许修改账号")
|
||||
}
|
||||
async fn get_quota(&self, account: &ProviderAccountId) -> Result<ProviderQuota, AdminError> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
self.reads.fetch_add(1, Ordering::SeqCst);
|
||||
assert_eq!(account.as_str(), "acct_facts");
|
||||
Ok(ProviderQuota {
|
||||
credits: None,
|
||||
credits: Some(ProviderQuotaCredits {
|
||||
has_credits: true,
|
||||
unlimited: false,
|
||||
balance: Some("123456789.123456789".into()),
|
||||
}),
|
||||
plan_type: Some("private plan".into()),
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
@@ -129,17 +145,24 @@ impl PluginAccountAccess for Facts {
|
||||
label: "private label".into(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
role: Some(ProviderQuotaWindowRole::Primary),
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: Some(604800),
|
||||
used_percent: None,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
limit_reached: true,
|
||||
provider_data: Some(ProviderDocument::new(
|
||||
gateway_core::account::OpaqueProviderData::new(
|
||||
json!({"private":"provider-internal-fixture"})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
),
|
||||
)),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -157,11 +180,10 @@ async fn management_facts_validate_queries_without_permission_declarations() {
|
||||
Default::default(),
|
||||
)
|
||||
.await;
|
||||
let facts = Arc::new(Facts(
|
||||
AtomicUsize::new(0),
|
||||
AtomicUsize::new(0),
|
||||
email.map(str::to_owned),
|
||||
));
|
||||
let facts = Arc::new(Facts {
|
||||
email: email.map(str::to_owned),
|
||||
..Default::default()
|
||||
});
|
||||
let port: Arc<dyn PluginAccountAccess> = facts.clone();
|
||||
runtime.bind_account_ports(&port).unwrap();
|
||||
{
|
||||
@@ -172,6 +194,7 @@ async fn management_facts_validate_queries_without_permission_declarations() {
|
||||
"data_queries":[
|
||||
{"method":"host.data.accounts.list","query":{"provider_id":"openai","cursor":"acct_before","limit":1}},
|
||||
{"method":"host.data.quota.get","query":{"account_id":"acct_facts"}},
|
||||
{"method":"host.data.quota.details","query":{"account_id":"acct_facts"}},
|
||||
{"method":"host.data.accounts.list","query":{"limit":0}},
|
||||
{"method":"host.data.accounts.list","query":{"limit":65535}},
|
||||
{"method":"host.data.quota.get","query":{"account_id":"acct_facts","sql":"SELECT secret"}}
|
||||
@@ -209,7 +232,8 @@ async fn management_facts_validate_queries_without_permission_declarations() {
|
||||
.unwrap();
|
||||
let result: Value = serde_json::from_slice(&response.body).unwrap();
|
||||
|
||||
assert_eq!(facts.0.load(Ordering::SeqCst), 2, "{result}");
|
||||
assert_eq!(facts.reads.load(Ordering::SeqCst), 3, "{result}");
|
||||
assert_eq!(facts.refreshes.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
result[0],
|
||||
json!({"schema_version":1,"accounts":[{
|
||||
@@ -227,7 +251,19 @@ async fn management_facts_validate_queries_without_permission_declarations() {
|
||||
json!({"schema_version":1,"account_id":"acct_facts","observed_at_ms":null,
|
||||
"windows":[{"key":"weekly","window_seconds":604800,"used_percent":null,"reset_at_ms":null}]})
|
||||
);
|
||||
for invalid in &result.as_array().unwrap()[2..] {
|
||||
let details: gateway_plugin_sdk::call::data::QuotaDetails =
|
||||
serde_json::from_value(result[2].clone()).unwrap();
|
||||
assert_eq!(details.observed_at_ms, None);
|
||||
assert_eq!(details.windows[0].used_percent, None);
|
||||
assert_eq!(details.windows[0].role.as_deref(), Some("primary"));
|
||||
assert_eq!(details.windows[0].label, "private label");
|
||||
assert!(details.limit_reached && details.windows[0].limit_reached);
|
||||
assert_eq!(
|
||||
details.credits.unwrap().balance.as_deref(),
|
||||
Some("123456789.123456789")
|
||||
);
|
||||
assert!(!result[2].to_string().contains("provider-internal-fixture"));
|
||||
for invalid in &result.as_array().unwrap()[3..] {
|
||||
assert_eq!(invalid["error"], "invalid_input");
|
||||
}
|
||||
|
||||
@@ -301,7 +337,7 @@ async fn quota_refresh_is_available_without_permission_declarations() {
|
||||
.unwrap();
|
||||
let result: Value = serde_json::from_slice(&response.body).unwrap();
|
||||
|
||||
assert_eq!(facts.1.load(Ordering::SeqCst), 4);
|
||||
assert_eq!(facts.refreshes.load(Ordering::SeqCst), 4);
|
||||
assert_eq!(result[0]["observed_at_ms"], Value::Null);
|
||||
assert_eq!(
|
||||
result[1],
|
||||
@@ -323,3 +359,149 @@ async fn quota_refresh_is_available_without_permission_declarations() {
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn management_queries(accounts: Arc<dyn PluginAccountAccess>, queries: Value) -> Value {
|
||||
let (cache, store, runtime) =
|
||||
super::setup_with_contributions(Contributions::from([crate::support::contribution(
|
||||
Capability::Management,
|
||||
vec![Stage::Management],
|
||||
vec![],
|
||||
vec![],
|
||||
)]))
|
||||
.await;
|
||||
runtime.bind_account_ports(&accounts).unwrap();
|
||||
store.snapshot.lock().unwrap().instances[0].configuration = json!({
|
||||
"management_registration":{"routes":[{"method":"GET","path":"facts","request_content_types":[],"response_content_types":["application/json"]}]},
|
||||
"data_queries":queries,
|
||||
"sdk_session":true,
|
||||
});
|
||||
let generation = ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let view = runtime.views(&generation).await.unwrap().remove(0);
|
||||
let response = runtime
|
||||
.handle(
|
||||
&generation,
|
||||
&view.target,
|
||||
PluginManagementRequest {
|
||||
request_context: None,
|
||||
headers: vec![],
|
||||
method: "GET".into(),
|
||||
path: "facts".into(),
|
||||
query: String::new(),
|
||||
content_type: None,
|
||||
body: vec![],
|
||||
request_id: "account-http-fixture".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = serde_json::from_slice(&response.body).unwrap();
|
||||
drop(generation);
|
||||
runtime.shutdown().await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
result
|
||||
}
|
||||
|
||||
async fn request_head(stream: &mut tokio::net::TcpStream) -> String {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.ends_with(b"\r\n\r\n") {
|
||||
assert!(bytes.len() < 16 * 1024);
|
||||
bytes.push(stream.read_u8().await.unwrap());
|
||||
}
|
||||
String::from_utf8(bytes).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_http_uses_the_selected_proxy_and_rejects_unavailable_account_versions() {
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
for scenario in [
|
||||
"plain",
|
||||
"explicit-auth",
|
||||
"rotated",
|
||||
"disabled",
|
||||
"proxy-failed",
|
||||
"missing",
|
||||
] {
|
||||
let target = wiremock::MockServer::start().await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let proxy = gateway_core::account::OutboundProxy::parse(&format!(
|
||||
"http://{}",
|
||||
listener.local_addr().unwrap()
|
||||
))
|
||||
.unwrap();
|
||||
let successful = matches!(scenario, "plain" | "explicit-auth");
|
||||
let proxy_task = if successful {
|
||||
Some(tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let request = request_head(&mut socket).await;
|
||||
assert!(request.starts_with("GET http://127.0.0.1:"));
|
||||
socket
|
||||
.write_all(
|
||||
b"HTTP/1.1 201 Created\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
request
|
||||
}))
|
||||
} else {
|
||||
drop(listener);
|
||||
None
|
||||
};
|
||||
let mut selected = account();
|
||||
selected.outbound_proxy = Some(proxy);
|
||||
selected.enabled = scenario != "disabled";
|
||||
if scenario == "rotated" {
|
||||
selected.credential_revision = Revision::new(2).unwrap();
|
||||
}
|
||||
let accounts: Arc<dyn PluginAccountAccess> = Arc::new(Facts {
|
||||
runtime_account: Some(selected),
|
||||
..Default::default()
|
||||
});
|
||||
let headers = if scenario == "explicit-auth" {
|
||||
vec![("authorization", b"Bearer plugin-fixture".to_vec())]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let results = management_queries(accounts, json!([{
|
||||
"method":gateway_plugin_sdk::call::host::ACCOUNT_HTTP,
|
||||
"params":{
|
||||
"account_id":if scenario == "missing" {"acct_missing"} else {"acct_facts"},
|
||||
"credential_revision":1,
|
||||
"request":{"method":"GET","url":format!("{}/facts", target.uri()),"headers":headers},
|
||||
},
|
||||
}])).await;
|
||||
if let Some(task) = proxy_task {
|
||||
assert_eq!(results[0]["status"], 201, "{scenario}: {results}");
|
||||
let request = tokio::time::timeout(std::time::Duration::from_secs(3), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.to_ascii_lowercase();
|
||||
assert!(
|
||||
request.starts_with("get http://127.0.0.1:") && request.contains("/facts http/1.1"),
|
||||
"{request}"
|
||||
);
|
||||
assert_eq!(
|
||||
request.contains("authorization:"),
|
||||
scenario == "explicit-auth"
|
||||
);
|
||||
if scenario == "explicit-auth" {
|
||||
assert!(request.contains("authorization: bearer plugin-fixture\r\n"));
|
||||
}
|
||||
} else {
|
||||
let expected = match scenario {
|
||||
"rotated" => "conflict",
|
||||
"disabled" => "permission_denied",
|
||||
"missing" => "rejected",
|
||||
_ => "upstream",
|
||||
};
|
||||
assert_eq!(results[0]["error"], expected, "{scenario}: {results}");
|
||||
}
|
||||
assert!(
|
||||
target.received_requests().await.unwrap().is_empty(),
|
||||
"{scenario} must not connect directly"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -881,6 +881,74 @@ async fn invalid_client_key_and_group_scopes_are_rejected_before_publish() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observer_call_load_tracks_the_active_lease_until_completion() {
|
||||
use gateway_admin::ports::plugins::PluginRuntimeDiagnostics;
|
||||
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let started = markers.path().join("started.jsonl");
|
||||
let completed = markers.path().join("completed.jsonl");
|
||||
let (cache, store, runtime) = setup(
|
||||
vec![(
|
||||
"tracked",
|
||||
serde_json::json!({
|
||||
"observation_started_marker": started,
|
||||
"observation_marker": completed,
|
||||
"observation_delay_ms": 200,
|
||||
}),
|
||||
vec![binding(EventKind::RequestCompleted, 0)],
|
||||
)],
|
||||
RpcLimits::default(),
|
||||
)
|
||||
.await;
|
||||
let snapshot = store.snapshot.lock().unwrap().clone();
|
||||
let generation = gateway_core::routing::extensions::ExtensionPreparationPort::prepare(
|
||||
&runtime,
|
||||
ConfigRevision::new(1).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let plan = runtime.observer_registry().resolve(&generation).unwrap();
|
||||
plan.dispatch(
|
||||
generation.clone(),
|
||||
observation("req_tracked", RequestObservationOutcome::Succeeded)
|
||||
.with_provider(ProviderKind::new("openai").unwrap()),
|
||||
);
|
||||
wait_for_lines(&started, 1).await;
|
||||
let read_load = || async {
|
||||
runtime
|
||||
.runtime_diagnostics(&snapshot, Some(1), Some(&generation))
|
||||
.await
|
||||
.unwrap()
|
||||
.remove("tracked")
|
||||
.unwrap()
|
||||
.load
|
||||
.expect("published session has a load snapshot")
|
||||
};
|
||||
let load = read_load().await;
|
||||
assert_eq!(load.active_calls, 1);
|
||||
assert_eq!(load.executing_calls, 1);
|
||||
assert_eq!(load.queued_calls, 0);
|
||||
assert_eq!(load.pending_callbacks, 0);
|
||||
assert_eq!(load.active_streams, 0);
|
||||
|
||||
wait_for_lines(&completed, 1).await;
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let load = read_load().await;
|
||||
if load.active_calls == 0 && load.executing_calls == 0 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("completed calls release their observed resource occupancy");
|
||||
drop(plan);
|
||||
drop(generation);
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observer_backpressure_drops_excess_and_inflight_dispatch_keeps_generation_alive() {
|
||||
let marker_directory = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -41,6 +41,24 @@ async fn http_plugin_with_policy(
|
||||
PluginRuntime,
|
||||
ExtensionSetReference,
|
||||
gateway_core::engine::middleware::FrozenMiddlewarePlan,
|
||||
) {
|
||||
http_plugin_with_configuration(
|
||||
serde_json::json!({"http":true,"mode":mode}),
|
||||
limits,
|
||||
failure_policy,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn http_plugin_with_configuration(
|
||||
configuration: serde_json::Value,
|
||||
limits: RpcLimits,
|
||||
failure_policy: PluginFailurePolicy,
|
||||
) -> (
|
||||
tempfile::TempDir,
|
||||
PluginRuntime,
|
||||
ExtensionSetReference,
|
||||
gateway_core::engine::middleware::FrozenMiddlewarePlan,
|
||||
) {
|
||||
let worker = std::fs::read(env!("CARGO_BIN_EXE_gateway-plugin-test-middleware")).unwrap();
|
||||
let package = crate::support::package_with_contributions(
|
||||
@@ -55,7 +73,7 @@ async fn http_plugin_with_policy(
|
||||
let (cache, runtime) = setup_package_with_limits(
|
||||
vec![InstanceFixture {
|
||||
id: "http-plugin",
|
||||
configuration: serde_json::json!({"http":true,"mode":mode}),
|
||||
configuration,
|
||||
bindings: vec![binding(MIDDLEWARE_CONTRIBUTION, "http", 0, failure_policy)],
|
||||
}],
|
||||
package,
|
||||
|
||||
+8
-13
@@ -64,14 +64,11 @@ async fn session_model_and_network_callbacks_outlive_handshake_and_each_executio
|
||||
let key = format!("key_{}", uuid::Uuid::new_v4().simple());
|
||||
environment.client_key(&key, "sk-session-fixture").await;
|
||||
environment
|
||||
.install_plugin(serde_json::json!({"http":true,"mode":"callbacks"}))
|
||||
.await;
|
||||
let (runtime, bundle) = environment
|
||||
.runtime_with_limits(RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(500),
|
||||
..RpcLimits::default()
|
||||
})
|
||||
.install_plugin(
|
||||
serde_json::json!({"http":true,"mode":"callbacks", "call_budgets_ms":{"http":500}}),
|
||||
)
|
||||
.await;
|
||||
let (runtime, bundle) = environment.runtime_with_limits(RpcLimits::default()).await;
|
||||
environment.store.start_command_line_writes().unwrap();
|
||||
let snapshot = bundle.snapshots().snapshot_for_diagnostics().unwrap();
|
||||
let plan = runtime
|
||||
@@ -185,12 +182,10 @@ async fn session_model_and_network_callbacks_outlive_handshake_and_each_executio
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_plugin_controls_upgrade_and_concurrent_duplex_with_large_binary_frames() {
|
||||
let (_cache, runtime, generation, plan) = http_plugin_with_limits(
|
||||
"upgrade",
|
||||
RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(500),
|
||||
..RpcLimits::default()
|
||||
},
|
||||
let (_cache, runtime, generation, plan) = http_plugin_with_configuration(
|
||||
serde_json::json!({"http":true,"mode":"upgrade", "call_budgets_ms":{"http":500}}),
|
||||
RpcLimits::default(),
|
||||
PluginFailurePolicy::Reject,
|
||||
)
|
||||
.await;
|
||||
let (send, incoming) = tokio::sync::mpsc::channel(1);
|
||||
|
||||
@@ -147,7 +147,8 @@ async fn sdk_budget_runtime(stage: Stage, count: usize) -> (tempfile::TempDir, P
|
||||
.enumerate()
|
||||
.map(|(order, id)| InstanceFixture {
|
||||
id,
|
||||
configuration: serde_json::json!({"mode":"map", "attempt":stage == Stage::Attempt}),
|
||||
configuration: serde_json::json!({"mode":"map", "attempt":stage == Stage::Attempt,
|
||||
"call_budgets_ms":{"request":300,"attempt":300}}),
|
||||
bindings: vec![binding(
|
||||
MIDDLEWARE_CONTRIBUTION,
|
||||
mount,
|
||||
@@ -156,15 +157,7 @@ async fn sdk_budget_runtime(stage: Stage, count: usize) -> (tempfile::TempDir, P
|
||||
)],
|
||||
})
|
||||
.collect();
|
||||
setup_package_with_limits(
|
||||
instances,
|
||||
package,
|
||||
RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(300),
|
||||
..RpcLimits::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
setup_package_with_limits(instances, package, RpcLimits::default()).await
|
||||
}
|
||||
|
||||
fn budget_request() -> MiddlewareRequest {
|
||||
|
||||
@@ -369,54 +369,85 @@ async fn retry_policy_preserves_decisions_and_leaves_replay_safety_to_core() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_policy_timeout_exhausts_the_chain_budget_and_delegates() {
|
||||
async fn retry_policy_combines_instance_budgets_with_one_request_bounded_chain() {
|
||||
use gateway_core::{
|
||||
engine::policy::{RetryDecision, RetryFacts},
|
||||
error::ProviderErrorKind,
|
||||
upstream::UpstreamSendState,
|
||||
};
|
||||
let records = tempfile::tempdir().unwrap();
|
||||
let second_marker = records.path().join("second.jsonl");
|
||||
let (_cache, runtime) = setup(
|
||||
vec![
|
||||
for (
|
||||
first_budget,
|
||||
first_delay,
|
||||
second_budget,
|
||||
second_delay,
|
||||
request_budget,
|
||||
expected,
|
||||
reached_second,
|
||||
) in [
|
||||
(100, 500, 1000, 100, 2000, RetryDecision::Stop, true),
|
||||
(1000, 100, 50, 300, 2000, RetryDecision::Delegate, true),
|
||||
(1000, 300, 1000, 800, 2000, RetryDecision::Delegate, true),
|
||||
(1000, 500, 1000, 0, 100, RetryDecision::Delegate, false),
|
||||
(3000, 2100, 3000, 0, 5000, RetryDecision::Stop, true),
|
||||
] {
|
||||
let records = tempfile::tempdir().unwrap();
|
||||
let second_marker = records.path().join("second.jsonl");
|
||||
let (_cache, runtime) = setup(vec![
|
||||
InstanceFixture {
|
||||
id: "retry-delayed",
|
||||
configuration: serde_json::json!({"retry_delay_ms":500,"retry_decision":{"decision":"stop"}}),
|
||||
|
||||
bindings: vec![binding("test.example.retryPolicy", "retry", 0, PluginFailurePolicy::Delegate)],
|
||||
configuration: serde_json::json!({
|
||||
"call_budgets_ms":{"retry":first_budget},
|
||||
"retry_delay_ms":first_delay,
|
||||
"retry_decision":{"decision":"delegate"},
|
||||
}),
|
||||
bindings: vec![binding(
|
||||
"test.example.retryPolicy",
|
||||
"retry",
|
||||
0,
|
||||
PluginFailurePolicy::Delegate,
|
||||
)],
|
||||
},
|
||||
InstanceFixture {
|
||||
id: "retry-next",
|
||||
configuration: serde_json::json!({"retry_marker":second_marker,"retry_decision":{"decision":"stop"}}),
|
||||
|
||||
bindings: vec![binding("test.example.retryPolicy", "retry", 1, PluginFailurePolicy::Delegate)],
|
||||
configuration: serde_json::json!({
|
||||
"call_budgets_ms":{"retry":second_budget},
|
||||
"retry_delay_ms":second_delay,
|
||||
"retry_marker":second_marker,
|
||||
"retry_decision":{"decision":"stop"},
|
||||
}),
|
||||
bindings: vec![binding(
|
||||
"test.example.retryPolicy",
|
||||
"retry",
|
||||
1,
|
||||
PluginFailurePolicy::Delegate,
|
||||
)],
|
||||
},
|
||||
],
|
||||
|
||||
).await;
|
||||
let context = policy_context(&runtime, prepare(&runtime).await, "req_retry_timeout");
|
||||
let decision = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
context.retry_decision(RetryFacts {
|
||||
attempt_index: NonZeroU32::MIN,
|
||||
provider: ProviderKind::new("openai").unwrap(),
|
||||
model: Some("gpt-test".into()),
|
||||
error_kind: ProviderErrorKind::RateLimited,
|
||||
upstream_status: Some(429),
|
||||
send_state: UpstreamSendState::NotSent,
|
||||
remaining_routing_attempts: 1,
|
||||
remaining_deadline: Duration::from_millis(100),
|
||||
retry_allowed: true,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("the shared deadline bounds the whole retry chain");
|
||||
assert_eq!(decision, RetryDecision::Delegate);
|
||||
assert!(
|
||||
!second_marker.exists(),
|
||||
"later policies cannot restart an exhausted budget"
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
])
|
||||
.await;
|
||||
let context = policy_context(&runtime, prepare(&runtime).await, "req_retry_timeout");
|
||||
let decision = tokio::time::timeout(
|
||||
Duration::from_secs(6),
|
||||
context.retry_decision(RetryFacts {
|
||||
attempt_index: NonZeroU32::MIN,
|
||||
provider: ProviderKind::new("openai").unwrap(),
|
||||
model: Some("gpt-test".into()),
|
||||
error_kind: ProviderErrorKind::RateLimited,
|
||||
upstream_status: Some(429),
|
||||
send_state: UpstreamSendState::NotSent,
|
||||
remaining_routing_attempts: 1,
|
||||
remaining_deadline: Duration::from_millis(request_budget),
|
||||
retry_allowed: true,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("重试链不能重置总预算");
|
||||
assert_eq!(
|
||||
decision, expected,
|
||||
"{first_budget}/{second_budget}/{request_budget}"
|
||||
);
|
||||
assert_eq!(second_marker.exists(), reached_second);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn marker_lines(path: &std::path::Path, count: usize) -> Vec<serde_json::Value> {
|
||||
|
||||
@@ -268,13 +268,18 @@ async fn outbound_next_wait_pauses_plugin_budget_and_cancel_releases_the_continu
|
||||
for cancel in [false, true] {
|
||||
let (cache, runtime) = runtime(
|
||||
vec![
|
||||
instance("outer", 0, serde_json::json!({})),
|
||||
instance("inner", 1, serde_json::json!({})),
|
||||
instance(
|
||||
"outer",
|
||||
0,
|
||||
serde_json::json!({"call_budgets_ms":{"outbound":300}}),
|
||||
),
|
||||
instance(
|
||||
"inner",
|
||||
1,
|
||||
serde_json::json!({"call_budgets_ms":{"outbound":300}}),
|
||||
),
|
||||
],
|
||||
RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(300),
|
||||
..RpcLimits::default()
|
||||
},
|
||||
RpcLimits::default(),
|
||||
)
|
||||
.await;
|
||||
let generation = prepare(&runtime).await;
|
||||
|
||||
@@ -136,6 +136,7 @@ async fn retry_callback_effects_survive_success_failure_and_cancellation() {
|
||||
"retry_http_return_marker":(outcome == "early_result").then_some(marker),
|
||||
"retry_service_request":is_service.then_some(service_request),
|
||||
"retry_service_marker":service_marker,
|
||||
"call_budgets_ms":{"retry":500},
|
||||
}))
|
||||
.await;
|
||||
let key = "sk-retry-effects-fixture";
|
||||
@@ -146,10 +147,7 @@ async fn retry_callback_effects_survive_success_failure_and_cancellation() {
|
||||
});
|
||||
let (runtime, core) = environment
|
||||
.runtime_with_providers(
|
||||
RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(500),
|
||||
..RpcLimits::default()
|
||||
},
|
||||
RpcLimits::default(),
|
||||
ProviderRegistry::new([provider.clone() as Arc<dyn Provider>]).unwrap(),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
use super::wait_until_unready;
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::plugins::instances::PluginInstanceRuntimeStatus;
|
||||
use gateway_admin::model::plugins::instances::{
|
||||
PluginInstanceRuntimeLoad, PluginInstanceRuntimeStatus,
|
||||
};
|
||||
use gateway_admin::ports::plugins::PluginPreparation;
|
||||
use gateway_core::routing::ConfigRevision;
|
||||
use gateway_core::routing::extensions::ExtensionPreparationPort;
|
||||
@@ -31,6 +33,17 @@ async fn diagnostics_distinguish_running_failed_preparation_and_draining_generat
|
||||
Some(snapshot.instances[0].artifact_sha256.as_str())
|
||||
);
|
||||
assert!(running.failure.is_none());
|
||||
assert_eq!(
|
||||
running.load,
|
||||
Some(PluginInstanceRuntimeLoad {
|
||||
active_calls: 0,
|
||||
executing_calls: 0,
|
||||
queued_calls: 0,
|
||||
resuming_calls: 0,
|
||||
pending_callbacks: 0,
|
||||
active_streams: 0,
|
||||
})
|
||||
);
|
||||
|
||||
let mut failed = snapshot.clone();
|
||||
failed.config_revision = Revision::new(2).unwrap();
|
||||
@@ -56,6 +69,7 @@ async fn diagnostics_distinguish_running_failed_preparation_and_draining_generat
|
||||
);
|
||||
assert!(failed_runtime.failure.is_some());
|
||||
assert_eq!(failed_runtime.actual_revision, Some(1));
|
||||
assert_eq!(failed_runtime.load, running.load);
|
||||
|
||||
let mut disabled = failed;
|
||||
disabled.config_revision = Revision::new(3).unwrap();
|
||||
@@ -105,6 +119,7 @@ async fn preparing_diagnostics_never_wait_for_the_serialized_prepare_io() {
|
||||
diagnostics["instance-one"].status,
|
||||
PluginInstanceRuntimeStatus::Preparing
|
||||
);
|
||||
assert!(diagnostics["instance-one"].load.is_none());
|
||||
|
||||
let candidate = preparation.await.expect("delayed candidate");
|
||||
drop(candidate);
|
||||
|
||||
@@ -552,3 +552,55 @@ async fn a_host_version_mismatch_still_restores_the_enabled_plugin() {
|
||||
runtime.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn declared_registration_budget_is_not_cut_short_by_process_preparation_timeout() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
snapshot.instances[0].configuration = serde_json::json!({
|
||||
"registration_delay_ms":31_000,
|
||||
"call_budgets_ms":{"registration":40_000},
|
||||
});
|
||||
let candidate = tokio::time::timeout(
|
||||
Duration::from_secs(45),
|
||||
PluginPreparation::prepare(&runtime, snapshot),
|
||||
)
|
||||
.await
|
||||
.expect("registration stays bounded by its declared budget")
|
||||
.expect("ready process may register beyond the startup control deadline");
|
||||
assert!(candidate.is_ready());
|
||||
drop(candidate);
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cancels_long_registration_and_reclaims_the_unpublished_candidate() {
|
||||
let (cache, store, runtime) = super::super::setup().await;
|
||||
let runtime = std::sync::Arc::new(runtime);
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let started = markers.path().join("registration");
|
||||
let mut snapshot = store.snapshot.lock().unwrap().clone();
|
||||
snapshot.instances[0].configuration = serde_json::json!({
|
||||
"registration_started_marker":started,
|
||||
"registration_delay_ms":600_000,
|
||||
"call_budgets_ms":{"registration":600_000},
|
||||
});
|
||||
let preparing = tokio::spawn({
|
||||
let runtime = std::sync::Arc::clone(&runtime);
|
||||
async move { PluginPreparation::prepare(runtime.as_ref(), snapshot).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !started.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(3), runtime.shutdown())
|
||||
.await
|
||||
.expect(
|
||||
"shutdown must cancel candidate preparation instead of waiting for its long budget",
|
||||
);
|
||||
assert!(preparing.await.unwrap().is_err());
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
//! 验证维护任务跨发布保活、通知合并及实例替换后的取消顺序
|
||||
|
||||
use gateway_admin::{
|
||||
model::{Revision, plugins::instances::PluginInstanceSnapshot},
|
||||
ports::plugins::PluginPreparation,
|
||||
};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
routing::{ConfigRevision, RuntimeSnapshot},
|
||||
runtime::RuntimeSnapshotHandle,
|
||||
task::{WorkerContribution, WorkerDaemonContext, WorkerRunnable},
|
||||
};
|
||||
use gateway_plugin_runtime::PluginRuntime;
|
||||
use gateway_plugin_sdk::{Capability, Contributions, Stage};
|
||||
use serde_json::{Value, json};
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
async fn publish(
|
||||
runtime: &PluginRuntime,
|
||||
snapshots: &RuntimeSnapshotHandle,
|
||||
instances: PluginInstanceSnapshot,
|
||||
) {
|
||||
let revision = ConfigRevision::new(instances.config_revision.get()).unwrap();
|
||||
let prepared = PluginPreparation::prepare(runtime, instances)
|
||||
.await
|
||||
.unwrap();
|
||||
snapshots.publish(
|
||||
RuntimeSnapshot::new(
|
||||
revision,
|
||||
gateway_core::settings::SettingsValues::new(1, "smart", Default::default(), None, None),
|
||||
vec![],
|
||||
vec![],
|
||||
vec![],
|
||||
)
|
||||
.unwrap()
|
||||
.with_extensions(Some(prepared)),
|
||||
);
|
||||
}
|
||||
|
||||
fn records(path: &Path) -> Vec<Value> {
|
||||
std::fs::read_to_string(path)
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str(line).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn wait_records(path: &Path, count: usize) -> Vec<Value> {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let entries = records(path);
|
||||
if entries.len() >= count {
|
||||
return entries;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("missing maintenance records: {error}; {:?}", records(path)))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unrelated_publications_preserve_inflight_maintenance_and_replacement_waits_for_cancel() {
|
||||
let (cache, store, runtime) = super::super::setup_with_contributions(Contributions::from([
|
||||
crate::support::contribution(
|
||||
Capability::Maintenance,
|
||||
vec![Stage::Maintenance],
|
||||
vec![],
|
||||
vec![],
|
||||
),
|
||||
]))
|
||||
.await;
|
||||
let runtime = Arc::new(runtime);
|
||||
let markers = tempfile::tempdir().unwrap();
|
||||
let mut instances = store.snapshot.lock().unwrap().clone();
|
||||
let template = instances.instances[0].clone();
|
||||
instances.instances = ["a", "b"]
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let mut instance = template.clone();
|
||||
instance.id = id.into();
|
||||
instance.configuration = json!({
|
||||
"maintenance_marker":markers.path().join(id),
|
||||
"execution_cancelled_marker":markers.path().join(id),
|
||||
"maintenance_wait_marker":markers.path().join(format!("release-{id}")),
|
||||
"call_context_marker":markers.path().join(format!("context-{id}")),
|
||||
"call_budgets_ms":{"maintenance":600_000},
|
||||
});
|
||||
instance
|
||||
})
|
||||
.collect();
|
||||
let snapshots = RuntimeSnapshotHandle::default();
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let WorkerContribution::Registration(registration) =
|
||||
runtime.maintenance_worker(snapshots.clone()).unwrap()
|
||||
else {
|
||||
panic!("maintenance worker registration")
|
||||
};
|
||||
let WorkerRunnable::Daemon { task, .. } = registration.runnable else {
|
||||
panic!("maintenance daemon")
|
||||
};
|
||||
let stop = CancellationToken::new();
|
||||
let running = tokio::spawn({
|
||||
let stop = stop.clone();
|
||||
async move { task.run(WorkerDaemonContext::new(stop)).await.unwrap() }
|
||||
});
|
||||
let a = markers.path().join("a");
|
||||
let b = markers.path().join("b");
|
||||
assert_eq!(wait_records(&a, 1).await[0]["phase"], "start");
|
||||
assert_eq!(wait_records(&b, 1).await[0]["phase"], "start");
|
||||
let contexts = records(&markers.path().join("context-a"));
|
||||
let maintenance = contexts
|
||||
.iter()
|
||||
.find(|entry| entry["method"] == "plugin.reconcile")
|
||||
.expect("maintenance call context");
|
||||
assert_eq!(maintenance["context"]["timeout_ms"], 600_000);
|
||||
|
||||
instances.config_revision = Revision::new(2).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
let changed = wait_records(&b, 3).await;
|
||||
assert!(
|
||||
changed[1].get("call_id").is_some(),
|
||||
"旧维护必须先完成取消确认"
|
||||
);
|
||||
assert_eq!(changed[2]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "无关实例发布不得重启 A 的在途维护");
|
||||
|
||||
instances.config_revision = Revision::new(3).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = false;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert!(wait_records(&b, 4).await[3].get("call_id").is_some());
|
||||
instances.config_revision = Revision::new(4).unwrap();
|
||||
instances.instances[1].revision = instances.config_revision;
|
||||
instances.instances[1].enabled = true;
|
||||
publish(&runtime, &snapshots, instances.clone()).await;
|
||||
assert_eq!(wait_records(&b, 5).await[4]["phase"], "start");
|
||||
assert_eq!(records(&a).len(), 1, "停用与恢复 B 不得取消 A");
|
||||
|
||||
// 多次发布通知合并为一次后续对账,首轮仍使用原调用 ID 完成
|
||||
std::fs::write(markers.path().join("release-a"), []).unwrap();
|
||||
let completed = wait_records(&a, 4).await;
|
||||
assert_eq!(completed[1]["phase"], "done");
|
||||
assert_eq!(completed[1]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed[2]["phase"], "start");
|
||||
assert_eq!(completed[3]["phase"], "done");
|
||||
assert_ne!(completed[2]["id"], completed[0]["id"]);
|
||||
assert_eq!(completed.len(), 4);
|
||||
|
||||
stop.cancel();
|
||||
running.await.unwrap();
|
||||
assert!(wait_records(&b, 6).await[5].get("call_id").is_some());
|
||||
snapshots.suspend();
|
||||
runtime.shutdown().await;
|
||||
super::super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user