Compare commits

...
144 Commits
Author SHA1 Message Date
ZyyandCodex 3510a45742 chore: bump version to 3.3.0
Co-authored-by: Codex <noreply@openai.com>
2026-08-24 04:07:37 +08:00
ZyyandCodex 85eb283b40 docs(readme): streamline project introduction
Co-authored-by: Codex <noreply@openai.com>
2026-08-24 04:06:49 +08:00
ZyyandCodex b8e1c5077d docs(theme): document runtime theme architecture
Describe the token model, preset themes, runtime stylesheet, isolated editor preview, naming conventions, and extension workflow.

Co-authored-by: Codex <noreply@openai.com>
2026-08-24 04:04:59 +08:00
ZyyandCodex 8422240967 feat(theme): add runtime token theme editor
Introduce Seed-to-Component token derivation, isolated live previews, semantic base component theming, and responsive theme controls.

Co-authored-by: Codex <noreply@openai.com>
2026-08-24 04:04:22 +08:00
ZyyandCodex fbd922117e fix(accounts): return persisted OAuth account ID
Co-authored-by: Codex <noreply@openai.com>
2026-08-22 19:37:03 +08:00
ZyyandCodex 941c4d57fa chore: bump version to 3.2.16
Co-authored-by: Codex <noreply@openai.com>
2026-08-22 09:28:34 +08:00
ZyyandCodex cf046ad9a4 fix(admin): close quota reset modal on success
Co-authored-by: Codex <noreply@openai.com>
2026-08-22 09:25:05 +08:00
ZyyandCodex 7699a954ac fix(admin): format account counts with separators
Co-authored-by: Codex <noreply@openai.com>
2026-08-22 09:21:11 +08:00
ZyyandCodex 2c31708d1b feat(admin): allow selecting quota reset credits
Co-authored-by: Codex <noreply@openai.com>
2026-08-22 09:19:20 +08:00
ZyyandCodex 80f338ce01 fix(admin): show raw account counts
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 23:12:54 +08:00
ZyyandCodex 299088a6e7 chore: bump version to 3.2.15
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 20:00:59 +08:00
ZyyandCodex a23bfca848 fix(admin): normalize OAuth credential formats
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 14:57:16 +08:00
ZyyandCodex 8951a64a05 fix(admin): raise billing abbreviation threshold
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 14:15:22 +08:00
ZyyandCodex eb20561893 fix(admin): align xAI quota usage displays
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 13:58:02 +08:00
ZyyandCodex a33637a99e fix(openai): use Codex image endpoint paths
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 13:02:40 +08:00
ZyyandCodex 97cd7a7097 chore: bump version to 3.2.14
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 01:11:57 +08:00
ZyyandCodex b0fe132736 docs: refresh architecture and project guides
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 01:01:07 +08:00
ZyyandCodex e2c75d1054 feat(admin): add direct token import and polish displays
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 00:33:26 +08:00
ZyyandCodex 94f298927d feat(openai): add active quota reset credits
Co-authored-by: Codex <noreply@openai.com>
2026-08-21 00:33:21 +08:00
ZyyandCodex 3be827204f fix(openai): keep image routes model-agnostic
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 19:05:14 +08:00
ZyyandCodex 227a1174b9 feat(openai): proxy image generation and edits
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 17:41:15 +08:00
ZyyandCodex c0defcb6df fix(admin): prefer account-wide dashboard quota
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 16:06:05 +08:00
ZyyandCodex 53739ebe5c chore: bump version to 3.2.13
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 15:20:29 +08:00
ZyyandCodex 7f420257a6 fix(api): keep provider degradation out of healthz
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 15:14:28 +08:00
ZyyandCodex 4f85209dc9 fix(config): centralize runtime data paths
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 15:14:28 +08:00
ZyyandCodex cd89e2e8fc fix(admin): align quota metadata and usage layout
Co-authored-by: Codex <noreply@openai.com>
2026-08-20 15:14:28 +08:00
ZyyandCodex 0cb5a29699 chore: bump version to 3.2.12
Co-authored-by: Codex <noreply@openai.com>
2026-08-19 23:52:11 +08:00
ZyyandCodex 61788804b4 fix(openai): recover cross-account websocket continuations
Co-authored-by: Codex <noreply@openai.com>
2026-08-19 23:40:40 +08:00
ZyyandCodex 2751ea6aa4 fix(openai): preserve cross-account session recovery
Co-authored-by: Codex <noreply@openai.com>
2026-08-19 23:05:48 +08:00
ZyyandCodex 58fe66789c chore(deps): bump h2 to 0.4.16
Resolve RUSTSEC-2026-0258 reported by cargo audit.

Co-authored-by: Codex <noreply@openai.com>
2026-08-19 11:34:07 +08:00
ZyyandCodex 573b6b26db fix(openai): isolate quota buckets by limit id
Co-authored-by: Codex <noreply@openai.com>
2026-08-18 16:48:37 +08:00
ZyyandCodex d425428ebe chore: bump version to 3.2.11
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 20:02:02 +08:00
ZyyandCodex b26d3b386a fix(admin): align quota diagnostics and usage layout
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 20:00:29 +08:00
ZyyandCodex cf0b14c5fd chore: bump version to 3.2.10
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 15:40:35 +08:00
ZyyandCodex 056561de41 refactor(frontend): clarify component boundaries
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 15:33:36 +08:00
dependabot[bot] f2215df72e chore(deps): bump the minor-and-patch group in /frontend with 4 updates
Bumps the minor-and-patch group in /frontend with 4 updates: [@lucide/vue](https://github.com/lucide-icons/lucide/tree/HEAD/packages/vue), [pinia](https://github.com/vuejs/pinia), [eslint-plugin-vuejs-accessibility](https://github.com/vue-a11y/eslint-plugin-vuejs-accessibility) and [vue-tsc](https://github.com/vuejs/language-tools/tree/HEAD/packages/tsc).


Updates `@lucide/vue` from 1.30.0 to 1.31.0
- [Release notes](https://github.com/lucide-icons/lucide/releases)
- [Commits](https://github.com/lucide-icons/lucide/commits/1.31.0/packages/vue)

Updates `pinia` from 4.0.2 to 4.0.3
- [Release notes](https://github.com/vuejs/pinia/releases)
- [Commits](https://github.com/vuejs/pinia/compare/v4.0.2...v4.0.3)

Updates `eslint-plugin-vuejs-accessibility` from 2.5.0 to 2.6.0
- [Release notes](https://github.com/vue-a11y/eslint-plugin-vuejs-accessibility/releases)
- [Changelog](https://github.com/vue-a11y/eslint-plugin-vuejs-accessibility/blob/main/CHANGELOG.md)
- [Commits](https://github.com/vue-a11y/eslint-plugin-vuejs-accessibility/compare/v2.5.0...v2.6.0)

Updates `vue-tsc` from 3.3.9 to 3.3.10
- [Release notes](https://github.com/vuejs/language-tools/releases)
- [Changelog](https://github.com/vuejs/language-tools/blob/master/CHANGELOG.md)
- [Commits](https://github.com/vuejs/language-tools/commits/v3.3.10/packages/tsc)

---
updated-dependencies:
- dependency-name: "@lucide/vue"
  dependency-version: 1.31.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: pinia
  dependency-version: 4.0.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: eslint-plugin-vuejs-accessibility
  dependency-version: 2.6.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: vue-tsc
  dependency-version: 3.3.10
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-17 14:05:19 +08:00
dependabot[bot] 72f8957163 chore(deps): bump the minor-and-patch group in /backend with 5 updates
Bumps the minor-and-patch group in /backend with 5 updates:

| Package | From | To |
| --- | --- | --- |
| [thiserror](https://github.com/dtolnay/thiserror) | `2.0.19` | `2.0.20` |
| [async-trait](https://github.com/dtolnay/async-trait) | `0.1.91` | `0.1.92` |
| [futures](https://github.com/rust-lang/futures-rs) | `0.3.33` | `0.3.34` |
| [aws-smithy-types](https://github.com/smithy-lang/smithy-rs) | `1.6.1` | `1.6.2` |
| [cookie](https://github.com/SergioBenitez/cookie-rs) | `0.18.1` | `0.18.2` |


Updates `thiserror` from 2.0.19 to 2.0.20
- [Release notes](https://github.com/dtolnay/thiserror/releases)
- [Commits](https://github.com/dtolnay/thiserror/compare/2.0.19...2.0.20)

Updates `async-trait` from 0.1.91 to 0.1.92
- [Release notes](https://github.com/dtolnay/async-trait/releases)
- [Commits](https://github.com/dtolnay/async-trait/compare/0.1.91...0.1.92)

Updates `futures` from 0.3.33 to 0.3.34
- [Release notes](https://github.com/rust-lang/futures-rs/releases)
- [Changelog](https://github.com/rust-lang/futures-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.33...0.3.34)

Updates `aws-smithy-types` from 1.6.1 to 1.6.2
- [Release notes](https://github.com/smithy-lang/smithy-rs/releases)
- [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/smithy-lang/smithy-rs/commits)

Updates `cookie` from 0.18.1 to 0.18.2
- [Changelog](https://github.com/rwf2/cookie-rs/blob/0.18.2/CHANGELOG.md)
- [Commits](https://github.com/SergioBenitez/cookie-rs/compare/0.18.1...0.18.2)

---
updated-dependencies:
- dependency-name: thiserror
  dependency-version: 2.0.20
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: async-trait
  dependency-version: 0.1.92
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: futures
  dependency-version: 0.3.34
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: aws-smithy-types
  dependency-version: 1.6.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: cookie
  dependency-version: 0.18.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-17 14:05:19 +08:00
dependabot[bot] 73f0c15790 chore(deps): bump github/codeql-action/upload-sarif
Bumps the minor-and-patch group with 1 update: [github/codeql-action/upload-sarif](https://github.com/github/codeql-action).


Updates `github/codeql-action/upload-sarif` from 4.37.6 to 4.37.7
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/5595ccaf912efad79be6eef63a5619ff05969be3...ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd)

---
updated-dependencies:
- dependency-name: github/codeql-action/upload-sarif
  dependency-version: 4.37.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-17 14:05:19 +08:00
ZyyandCodex 0558693949 fix(openai): harden refresh recovery handling
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 13:37:54 +08:00
ZyyandCodex 2a9cb02cc9 fix(openai): align OAuth refresh error handling
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 12:30:05 +08:00
ZyyandCodex 5ce36931a8 chore: bump version to 3.2.9
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 11:38:49 +08:00
ZyyandCodex 0d826d9eba fix(openai): align OAuth token refresh
Co-authored-by: Codex <noreply@openai.com>
2026-08-17 11:14:03 +08:00
ZyyandCodex 304242957a fix(accounts): preserve upstream error messages
Co-authored-by: Codex <noreply@openai.com>
2026-08-16 13:28:28 +08:00
ZyyandCodex ef37ae8d9e chore: bump version to 3.2.8
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 23:31:04 +08:00
ZyyandCodex fc13d4f2b6 fix(frontend): restore responsive component behavior
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 20:38:48 +08:00
ZyyandCodex 78f9c0c800 chore: bump version to 3.2.7
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 18:54:52 +08:00
ZyyandCodex a3bc7bc445 fix(frontend): increase account table row height
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 18:54:02 +08:00
ZyyandCodex 0132114285 fix(openai): let upstream validate catalog models
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 18:45:06 +08:00
ZyyandCodex 5970efbb7c fix(container): refresh runtime packages before scans
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 18:34:41 +08:00
ZyyandCodex 7bf28b571d chore: bump version to 3.2.6
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 15:47:32 +08:00
ZyyandCodex bc3bd6bc43 refactor(frontend): unify component design system
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 15:39:31 +08:00
ZyyandCodex 9acfab0a86 chore(frontend): enforce template interpolation layout
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 12:31:10 +08:00
ZyyandCodex 3e5a84d2a0 fix(frontend): refine dashboard capacity display
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 12:31:06 +08:00
ZyyandCodex 92948f51a9 fix(frontend): remove table bottom gap
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 11:59:58 +08:00
ZyyandCodex 776637fccb chore: bump version to 3.2.5
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 01:52:54 +08:00
ZyyandCodex 82acb08d26 fix(frontend): restore diagnostic table scrolling
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 01:51:56 +08:00
ZyyandCodex 7744e1b0b4 fix(accounts): preserve rolling window usage
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 01:28:44 +08:00
ZyyandCodex 0b8a1add18 feat(frontend): refresh account setup defaults and copy
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 01:09:31 +08:00
ZyyandCodex 2dd60c08b6 chore: bump version to 3.2.4
Co-authored-by: Codex <noreply@openai.com>
2026-08-15 00:29:44 +08:00
ZyyandCodex cce9880937 fix(xai): align Grok behavior with sub2api
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 23:36:12 +08:00
Zyy b8ea1cd076 fix(core): normalize account scheduling load 2026-08-14 23:27:06 +08:00
ZyyandCodex 4117840f75 fix(xai): align OAuth and Codex compatibility
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 22:24:08 +08:00
ZyyandCodex f642764f34 fix(openai): align billing with official pricing
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 21:14:36 +08:00
ZyyandCodex 482e192e2d fix(openai): align desktop oauth identity
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 17:43:07 +08:00
ZyyandCodex ee51a07de4 fix(openai): align desktop wire profile
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 16:13:11 +08:00
ZyyandCodex bf23658aaf chore: bump version to 3.2.3
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 01:25:44 +08:00
ZyyandCodex bc4728332e fix(frontend): polish dashboard and table interactions
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 01:24:49 +08:00
ZyyandCodex 2cb5cda077 fix(quota): report exhausted usage as full
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 01:07:20 +08:00
ZyyandCodex 9d90dedb81 fix(frontend): refresh account data without table loading
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 01:07:12 +08:00
ZyyandCodex 6074cff68c chore: bump version to 3.2.2
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:43:13 +08:00
ZyyandCodex 6ab8e7d0a6 fix(frontend): grade model success rate colors
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:42:20 +08:00
ZyyandCodex 17f9ef8915 fix(observability): omit inactive accounts from daily usage
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:39:59 +08:00
ZyyandCodex 15e0a28bd0 fix(frontend): refine refresh loading and group alignment
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:39:50 +08:00
ZyyandCodex 58bc438992 chore: bump version to 3.2.1
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:13:58 +08:00
ZyyandCodex b04616f59f refactor(migrations): squash schema into initial baseline
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:13:09 +08:00
ZyyandCodex 82ba4fcbba fix(observability): align dashboard and quota window usage
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:11:56 +08:00
ZyyandCodex 819f428ac3 fix(frontend): align loading and daily usage display
Co-authored-by: Codex <noreply@openai.com>
2026-08-14 00:10:13 +08:00
ZyyandCodex 35f28e8aab chore: bump version to 3.2.0
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 22:43:48 +08:00
ZyyandCodex 65645c7403 fix(accounts): gate quota recovery and add manual reset
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 22:43:03 +08:00
ZyyandCodex 71159d5633 refactor(account-groups): remove unused members endpoint
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 17:28:43 +08:00
ZyyandCodex f2ebb09e34 docs: describe group routing and account scheduling
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 16:58:36 +08:00
ZyyandCodex 07acc67ce6 feat(frontend): add account group management
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 16:58:36 +08:00
ZyyandCodex 01ad8bba8a feat(routing): add account groups and weighted scheduling
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 16:58:36 +08:00
ZyyandCodex 986a128d15 fix(observability): separate live status from history range
Co-authored-by: Codex <noreply@openai.com>
2026-08-13 00:26:10 +08:00
ZyyandCodex 6cad2bb6a1 fix(openai): align account import with official token claims
Co-authored-by: Codex <noreply@openai.com>
2026-08-12 23:51:26 +08:00
ZyyandCodex 2b8b3de0e4 refactor(accounts): unify five-state account status
Co-authored-by: Codex <noreply@openai.com>
2026-08-12 23:50:31 +08:00
ZyyandCodex 4127886584 chore: bump version to 3.1.21
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 17:42:54 +08:00
ZyyandCodex ee3e5db6c2 fix(accounts): isolate model-specific quota usage
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 17:42:02 +08:00
ZyyandCodex 153cb67b9c refactor(login): refine panel details
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 17:21:31 +08:00
ZyyandCodex f370d355b6 fix(accounts): align quota ordering and summaries
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 17:21:23 +08:00
ZyyandCodex a31dad992c fix(frontend): stabilize button loading indicator
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 17:21:16 +08:00
ZyyandCodex 55a88f5f6b chore: bump version to 3.1.20
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 15:13:40 +08:00
ZyyandCodex 5d90a92582 fix(logging): rename OAuth recovery log file
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 15:06:40 +08:00
ZyyandCodex 93abe8efcd fix(logging): generalize OAuth recovery logs
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 14:55:59 +08:00
ZyyandCodex f4b75c42ed fix(openai): isolate quota and recovery signals
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 13:37:16 +08:00
ZyyandCodex 36ff038c78 chore: bump version to 3.1.19
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 12:17:28 +08:00
ZyyandCodex 545009590c fix(openai): persist authoritative quota exhaustion
Co-authored-by: Codex <noreply@openai.com>
2026-08-11 12:16:34 +08:00
ZyyandCodex 5b7b9b21ee fix(openai): require low usage for quota recovery
Co-authored-by: Codex <noreply@openai.com>
2026-08-10 15:55:58 +08:00
ZyyandCodex 44a2a87b69 fix(frontend): center button loading spinner
Co-authored-by: Codex <noreply@openai.com>
2026-08-10 15:55:58 +08:00
ZyyandCodex 184e88083b chore: bump version to 3.1.18
Co-authored-by: Codex <noreply@openai.com>
2026-08-10 12:25:42 +08:00
ZyyandCodex 53b40aea46 chore(deps): ignore fingerprint-sensitive updates
Co-authored-by: Codex <noreply@openai.com>
2026-08-10 12:23:45 +08:00
ZyyandCodex aa328b6dbf fix(openai): restore official TLS fingerprint pins
Co-authored-by: Codex <noreply@openai.com>
2026-08-10 12:23:36 +08:00
dependabot[bot]andZyy 64231ff403 chore(deps): bump the minor-and-patch group in /backend with 4 updates (#37)
Bumps the minor-and-patch group in /backend with 4 updates: [base64](https://github.com/marshallpierce/rust-base64), [aws-sdk-s3](https://github.com/awslabs/aws-sdk-rust), [hyper-rustls](https://github.com/rustls/hyper-rustls) and [rustls](https://github.com/rustls/rustls).


Updates `base64` from 0.23.0 to 0.23.1
- [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md)
- [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.23.0...v0.23.1)

Updates `aws-sdk-s3` from 1.140.0 to 1.141.0
- [Release notes](https://github.com/awslabs/aws-sdk-rust/releases)
- [Commits](https://github.com/awslabs/aws-sdk-rust/commits)

Updates `hyper-rustls` from 0.27.7 to 0.27.9
- [Release notes](https://github.com/rustls/hyper-rustls/releases)
- [Commits](https://github.com/rustls/hyper-rustls/compare/v/0.27.7...v/0.27.9)

Updates `rustls` from 0.23.36 to 0.23.43
- [Release notes](https://github.com/rustls/rustls/releases)
- [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rustls/rustls/compare/v/0.23.36...v/0.23.43)

---
updated-dependencies:
- dependency-name: base64
  dependency-version: 0.23.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: aws-sdk-s3
  dependency-version: 1.141.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: hyper-rustls
  dependency-version: 0.27.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: rustls
  dependency-version: 0.23.43
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Zyy <39424122+zyycn@users.noreply.github.com>
2026-08-10 12:05:00 +08:00
dependabot[bot]andZyy d7b80a2066 chore(deps): bump the minor-and-patch group with 5 updates (#36)
Bumps the minor-and-patch group with 5 updates:

| Package | From | To |
| --- | --- | --- |
| [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) | `4.37.4` | `4.37.6` |
| [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) | `2.9.1` | `2.9.2` |
| [pnpm/action-setup](https://github.com/pnpm/action-setup) | `6.0.9` | `6.0.10` |
| [dorny/paths-filter](https://github.com/dorny/paths-filter) | `4.0.2` | `4.0.3` |
| [actions/attest](https://github.com/actions/attest) | `4.2.1` | `4.2.2` |


Updates `github/codeql-action/upload-sarif` from 4.37.4 to 4.37.6
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/f205ea1c3313d32999d8d6a48b4f6530d4437b38...5595ccaf912efad79be6eef63a5619ff05969be3)

Updates `Swatinem/rust-cache` from 2.9.1 to 2.9.2
- [Release notes](https://github.com/swatinem/rust-cache/releases)
- [Changelog](https://github.com/Swatinem/rust-cache/blob/master/CHANGELOG.md)
- [Commits](https://github.com/swatinem/rust-cache/compare/c19371144df3bb44fab255c43d04cbc2ab54d1c4...6323deb102c322ba6fcbdcafc7e3dddab59af2b6)

Updates `pnpm/action-setup` from 6.0.9 to 6.0.10
- [Release notes](https://github.com/pnpm/action-setup/releases)
- [Commits](https://github.com/pnpm/action-setup/compare/0ebf47130e4866e96fce0953f49152a61190b271...0977fd99725f1db4007ccb2928dbb4e90d06cc86)

Updates `dorny/paths-filter` from 4.0.2 to 4.0.3
- [Release notes](https://github.com/dorny/paths-filter/releases)
- [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md)
- [Commits](https://github.com/dorny/paths-filter/compare/7b450fff21473bca461d4b92ce414b9d0420d706...ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d)

Updates `actions/attest` from 4.2.1 to 4.2.2
- [Release notes](https://github.com/actions/attest/releases)
- [Changelog](https://github.com/actions/attest/blob/main/RELEASE.md)
- [Commits](https://github.com/actions/attest/compare/508db95dd578ae2727ebd6217d5ba78e4fbda05d...1e69f48acb82d1966a394da916b4c1698aa569d6)

---
updated-dependencies:
- dependency-name: github/codeql-action/upload-sarif
  dependency-version: 4.37.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: Swatinem/rust-cache
  dependency-version: 2.9.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: pnpm/action-setup
  dependency-version: 6.0.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: dorny/paths-filter
  dependency-version: 4.0.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: actions/attest
  dependency-version: 4.2.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Zyy <39424122+zyycn@users.noreply.github.com>
2026-08-10 12:01:38 +08:00
dependabot[bot] 08d63ea3f3 chore(deps): bump debian from 7b140f3 to abd67ff in /deploy (#39)
Bumps debian from `7b140f3` to `abd67ff`.

---
updated-dependencies:
- dependency-name: debian
  dependency-version: bookworm-slim
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-10 11:58:47 +08:00
dependabot[bot] ac9c4c85ad chore(deps): bump the minor-and-patch group in /frontend with 8 updates (#38)
Bumps the minor-and-patch group in /frontend with 8 updates:

| Package | From | To |
| --- | --- | --- |
| [@lucide/vue](https://github.com/lucide-icons/lucide/tree/HEAD/packages/vue) | `1.28.0` | `1.30.0` |
| [dompurify](https://github.com/cure53/DOMPurify) | `3.4.12` | `3.4.13` |
| [marked](https://github.com/markedjs/marked) | `18.0.7` | `18.0.9` |
| [vue](https://github.com/vuejs/core) | `3.5.40` | `3.5.41` |
| [@antfu/eslint-config](https://github.com/antfu/eslint-config) | `9.2.0` | `9.3.0` |
| [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) | `26.1.2` | `26.2.0` |
| [eslint](https://github.com/eslint/eslint) | `10.8.0` | `10.8.1` |
| [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) | `8.2.0` | `8.2.1` |


Updates `@lucide/vue` from 1.28.0 to 1.30.0
- [Release notes](https://github.com/lucide-icons/lucide/releases)
- [Commits](https://github.com/lucide-icons/lucide/commits/1.30.0/packages/vue)

Updates `dompurify` from 3.4.12 to 3.4.13
- [Release notes](https://github.com/cure53/DOMPurify/releases)
- [Commits](https://github.com/cure53/DOMPurify/compare/3.4.12...3.4.13)

Updates `marked` from 18.0.7 to 18.0.9
- [Release notes](https://github.com/markedjs/marked/releases)
- [Commits](https://github.com/markedjs/marked/compare/v18.0.7...v18.0.9)

Updates `vue` from 3.5.40 to 3.5.41
- [Release notes](https://github.com/vuejs/core/releases)
- [Changelog](https://github.com/vuejs/core/blob/main/CHANGELOG.md)
- [Commits](https://github.com/vuejs/core/compare/v3.5.40...v3.5.41)

Updates `@antfu/eslint-config` from 9.2.0 to 9.3.0
- [Release notes](https://github.com/antfu/eslint-config/releases)
- [Commits](https://github.com/antfu/eslint-config/compare/v9.2.0...v9.3.0)

Updates `@types/node` from 26.1.2 to 26.2.0
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

Updates `eslint` from 10.8.0 to 10.8.1
- [Release notes](https://github.com/eslint/eslint/releases)
- [Commits](https://github.com/eslint/eslint/compare/v10.8.0...v10.8.1)

Updates `vite` from 8.2.0 to 8.2.1
- [Release notes](https://github.com/vitejs/vite/releases)
- [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md)
- [Commits](https://github.com/vitejs/vite/commits/v8.2.1/packages/vite)

---
updated-dependencies:
- dependency-name: "@lucide/vue"
  dependency-version: 1.30.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: dompurify
  dependency-version: 3.4.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: marked
  dependency-version: 18.0.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: vue
  dependency-version: 3.5.41
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: "@antfu/eslint-config"
  dependency-version: 9.3.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: "@types/node"
  dependency-version: 26.2.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: eslint
  dependency-version: 10.8.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: vite
  dependency-version: 8.2.1
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-10 11:54:36 +08:00
ZyyandCodex 492c401599 fix(openai): preserve state on quota auth failures
Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Zyy <zyycn@users.noreply.github.com>
2026-08-10 11:46:19 +08:00
ZyyandCodex a063258a40 chore: bump version to 3.1.17
Co-authored-by: Codex <noreply@openai.com>
2026-08-09 15:14:06 +08:00
ZyyandCodex dc590f2b5f fix: make OAuth refresh margin runtime-only
Co-authored-by: Codex <noreply@openai.com>
2026-08-09 15:12:43 +08:00
ZyyandCodex ca52e2bfd9 chore: bump version to 3.1.16
Co-authored-by: Codex <noreply@openai.com>
2026-08-09 11:59:08 +08:00
ZyyandCodex 43604e36dc fix: correct first token timing
Co-authored-by: Codex <noreply@openai.com>
2026-08-07 18:56:13 +08:00
ZyyandCodex a4a997e351 fix: use lifetime dashboard card totals
Co-authored-by: Codex <noreply@openai.com>
2026-08-07 18:56:13 +08:00
ZyyandCodex 73ca463924 feat: distinguish pro plan badges
Co-authored-by: Codex <noreply@openai.com>
2026-08-07 18:56:13 +08:00
ZyyandCodex 9eeec2c78c chore: bump version to 3.1.15
Co-authored-by: Codex <noreply@openai.com>
2026-08-07 15:15:00 +08:00
ZyyandCodex 90890be2b4 fix(openai): persist OAuth tokens before quota sync
Align OAuth token persistence and ID token metadata handling with Codex, and retry initial quota observation asynchronously.

Co-authored-by: Codex <noreply@openai.com>
2026-08-07 14:59:42 +08:00
ZyyandCodex 2e4bab17a8 chore: bump version to 3.1.14
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 16:01:07 +08:00
ZyyandCodex b052e79a16 fix(openai): preserve imported token expiry
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 15:49:42 +08:00
ZyyandCodex 9775451e83 fix(web): preserve button labels while loading
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 15:27:00 +08:00
ZyyandCodex c22a2ad144 test(logging): move recovery filter coverage
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 15:14:26 +08:00
ZyyandCodex 7bd362d544 chore: bump version to 3.1.13
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 14:56:08 +08:00
ZyyandCodex e7036a616f fix(web): center button loading indicators
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 14:53:53 +08:00
ZyyandCodex 98652d4c68 fix(openai): write oauth recovery to app logs
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 14:22:13 +08:00
ZyyandCodex d0dc7ef92b fix(openai): complete import profile from access token
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 12:31:13 +08:00
ZyyandCodex 194fb20d5f chore: bump version to 3.1.12
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 11:14:32 +08:00
ZyyandCodex 30f528f084 fix(openai): simplify import and oauth validation
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 11:14:18 +08:00
ZyyandCodex 28d024ceda chore: bump version to 3.1.11
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 09:54:05 +08:00
ZyyandCodex 6b9c98c59f fix(openai): persist successful refresh tokens directly
Co-authored-by: Codex <noreply@openai.com>
2026-08-06 09:50:03 +08:00
ZyyandCodex 1294e50b76 chore: bump version to 3.1.10
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 12:22:46 +08:00
ZyyandCodex a0e540642b fix(oauth): bound refresh recovery and persist failure reasons
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 12:21:16 +08:00
ZyyandCodex 14d6a7de31 fix(dashboard): align summary with daily trend
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 10:46:14 +08:00
ZyyandCodex cd7b60a3ba chore: bump version to 3.1.9
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 00:03:39 +08:00
ZyyandCodex ebbed22f36 fix(openai): keep Codex wire OS exact
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 00:03:21 +08:00
ZyyandCodex f6e5d8f673 fix(openai): drop null secondary quota windows
Co-authored-by: Codex <noreply@openai.com>
2026-08-05 00:03:11 +08:00
ZyyandCodex bf1c6211b8 chore: bump version to 3.1.8
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 22:30:40 +08:00
ZyyandCodex d388fa344a fix(openai): drop placeholder secondary quota windows
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 22:30:04 +08:00
ZyyandCodex eabb61b08c fix(openai): restore Codex Core app-server identity
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 22:30:01 +08:00
ZyyandCodex a5c3438927 fix(openai): isolate thread-spawn session affinity
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 22:29:53 +08:00
ZyyandCodex b6948542bb fix: align Codex Desktop HTTP identity
- send the official Desktop HTTP user agent across OpenAI upstream paths
- keep the audited model catalog version separate from the wire identity
- remove CLI release drift and clarify the dashboard client identity

Co-authored-by: Codex <noreply@openai.com>
2026-08-04 18:36:03 +08:00
ZyyandCodex 0242bd3193 chore: bump version to 3.1.7
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 16:19:58 +08:00
ZyyandCodex 229ef48470 fix: harden backup verification and detail UI
- read every multipart backup payload to its requested length
- make remote verification failures diagnosable
- refine status details, loading icons, and attempt-chain presentation

Co-authored-by: Codex <noreply@openai.com>
2026-08-04 16:18:53 +08:00
ZyyandCodex 19c35ba901 chore: bump version to 3.1.6
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 15:12:29 +08:00
ZyyandCodex 3c06ae97e8 fix: reconcile quota, backup, and session behavior
- preserve confirmed quota exhaustion until its recorded reset advances
- normalize account quota presentation and subagent session affinity
- repair S3-compatible backup handling and related admin UI details

Co-authored-by: Codex <noreply@openai.com>
2026-08-04 15:12:15 +08:00
ZyyandCodex f34966e4c7 chore: bump version to 3.1.5
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 08:54:00 +08:00
ZyyandCodex e9ebf45640 fix(accounts): reconcile quota status and dashboard metrics
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 08:53:29 +08:00
492 changed files with 46280 additions and 18955 deletions
+5
View File
@@ -21,6 +21,11 @@ updates:
day: monday
time: '09:10'
timezone: Asia/Shanghai
# 固定官方 Codex Desktop 的 TLS 指纹;升级前必须抓取握手并比对扩展顺序。
ignore:
- dependency-name: hyper-rustls
- dependency-name: reqwest
- dependency-name: rustls
groups:
minor-and-patch:
update-types:
+4 -1
View File
@@ -55,6 +55,9 @@ jobs:
target: runtime
load: true
tags: ${{ env.CPR_IMAGE }}
no-cache-filters: |
runtime-base
runtime
cache-from: type=gha,scope=runtime-ci
cache-to: type=gha,mode=max,scope=runtime-ci
@@ -76,7 +79,7 @@ jobs:
${{ hashFiles('trivy-results.sarif') != '' &&
(github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository) }}
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7
with:
sarif_file: trivy-results.sarif
category: runtime-container
+2 -2
View File
@@ -106,7 +106,7 @@ jobs:
components: rustfmt,clippy
- name: Cache Rust
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: backend -> target
@@ -157,7 +157,7 @@ jobs:
ref: ${{ inputs.ref }}
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with:
version: 11.7.0
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
- name: Detect affected checks
id: filter
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
with:
filters: |
backend:
+5 -2
View File
@@ -375,7 +375,7 @@ jobs:
PY
- name: Generate release asset provenance
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-path: dist/release/*
@@ -468,6 +468,9 @@ jobs:
push: true
build-args: |
CPR_PREBUILT_DIR=dist/docker
no-cache-filters: |
runtime-base
runtime-prebuilt
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
provenance: mode=max
@@ -581,7 +584,7 @@ jobs:
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
- name: Generate container provenance attestation
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
subject-digest: ${{ steps.manifest.outputs.digest }}
+2 -2
View File
@@ -31,7 +31,7 @@ jobs:
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
- name: Cache Rust
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: backend -> target
@@ -52,7 +52,7 @@ jobs:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
with:
version: 11.7.0
+73 -70
View File
@@ -12,31 +12,32 @@
[![GHCR](https://img.shields.io/badge/GHCR-codex--proxy--rs-2496ED?logo=docker&logoColor=white&style=flat-square)](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
[![MIT](https://img.shields.io/badge/License-MIT-blue.svg?style=flat-square)](https://opensource.org/license/mit)
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [运维](#运维) · [部署](deploy/README.md) · [接口](docs/api.md) · [架构](docs/architecture.md)
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [运行维护](#运行维护) · [部署](deploy/README.md) · [接口](docs/api.md) · [架构](docs/architecture.md)
</div>
> [!NOTE]
> 只支持 OpenAI Responses API,不提供 `/v1/chat/completions`。
> 数据面支持 OpenAI Responses、Images 和模型目录协议,不提供 `/v1/chat/completions`。
## 能力
## 功能
| 领域 | 实现 |
| -------- | ----------------------------------------------------------------------------------------------------- |
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
| 领域 | 当前能力 |
| --- | --- |
| 数据面 | Responses JSON、SSE、WebSocket、review 子代理、Images 生成/编辑与模型目录 |
| Provider | 固定 `openai`、`xai` 两个 Provider,各自拥有 OAuth、credential、额度、目录与 transport |
| 路由 | Client Key 限定账号分组;按模型能力编译 Provider 候选;支持全局精确模型映射、会话亲和和安全 fallback |
| 透明边界 | OpenAI 保留未知请求字段及字段顺序,并原样转发 SSE、WebSocket 与 Images 业务字节;xAI 在 Provider 内转换协议 |
| 账号 | OAuth、OpenAI AT/RT、账号文件与 Agent Identity 导入,credential/额度刷新、账号恢复及主动额度重置卡 |
| 管理 | 账号与分组、Client Key、运行设置、用量/错误观测、S3/R2 备份、系统更新与回滚 |
| 计量 | Token、图片、费用、延迟、账号与 Provider 归因,并记录上游响应确认的实际 `service_tier` |
架构边界、状态所有权和请求不变量见 [架构文档](docs/architecture.md)。
## 快速开始
需要 Docker Engine 与 Docker Compose Plugin。
### 1. 准备
### 1. 准备配置和目录
```bash
git clone https://github.com/zyycn/codex-proxy-rs.git
@@ -56,16 +57,16 @@ openssl rand -hex 24
openssl rand -hex 24
```
把两个结果写入 `deploy/config.yaml`,并单独设置管理员初始密码:
把两个结果分别写入 `deploy/config.yaml`,再设置管理员初始密码:
| 配置 | 约束 |
| -------------------------- | ---------------------------- |
| `store.database.password` | 48 位十六进制 |
| `store.redis.password` | 48 位十六进制 |
| `admin.default_password` | 至少 12 位,不能包含 `$` |
| 配置 | 约束 |
| --- | --- |
| `store.database.password` | 48 位十六进制 |
| `store.redis.password` | 48 位十六进制 |
| `admin.default_password` | 至少 12 位,不能包含 `$` |
PostgreSQL、Redis 容器和应用通过同一份 `config.yaml` 使用这两个基础设施密码,不需要额外
导出环境变量。管理员密码只在首次创建管理员时生效。
PostgreSQL、Redis 和应用从同一份 `config.yaml` 读取基础设施密码,不需要额外导出环境变量。
管理员密码只在首次创建管理员时生效。
Linux 需要允许容器组写入运行目录:
@@ -83,37 +84,36 @@ docker compose -f deploy/compose.yaml up -d --no-build
curl -i http://127.0.0.1:8080/healthz
```
`204 No Content` 表示应用、PostgreSQL 与 Redis 均可用。管理端地址:
`204 No Content` 表示应用、PostgreSQL 与 Redis 均可用。管理端地址为
`http://127.0.0.1:8080`。
### 3. 初始化
### 3. 初始化网关
1. 使用 `admin@cpr.local` 与初始密码登录。
2. 在账号页选择对应 Provider:OpenAI 支持 OAuth、账号文件与 Agent Identity;xAI 支持 OAuth
和兼容账号文件导入。
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
2. 导入或授权 Provider 账号:OpenAI 支持 OAuth、逐行 AT/RT、OAuth JSON 与 Agent Identity;
xAI 支持 OAuth 和账号 JSON。
3. 创建账号分组并按需配置全局模型映射。未命中的模型名原样交给候选 Provider。
4. 创建 `sk_...` Client Key,选择可用账号分组,并设置速率和并发限制;空分组范围表示全部账号。
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
[账号接口](docs/api.md#5-账号)。
OpenAI RT 导入会先向官方 token endpoint 换取 AT;仅 AT 的账号不具备自动续期能力,
能否参与调度仍取决于 token 中可验证的身份与后续上游事实。账号导入和首次 OAuth 创建会尽力读取一次额度,
额度上游暂时失败不会回滚已提交的账号。
OpenAI 账号的主动重置卡只在用户打开相应弹窗或手工刷新时查询。消费成功后管理端重新查询卡片与额度;
系统不会通过修改本地 `reset_at` 伪造重置。完整账号合同见 [账号接口](docs/api.md#5-账号)。
> [!IMPORTANT]
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
> [!NOTE]
> `admin.default_password` 只在首次创建管理员时生效。
完整部署、权限和密码轮换规则见 [deploy/README.md](deploy/README.md)。
完整的权限、密码轮换、备份和升级规则见 [部署文档](deploy/README.md)。
## 客户端接入
| 配置 | 值 |
| -------- | ---------------------------------------- |
| Base URL | `http://127.0.0.1:8080/v1` |
| API Key | 管理端创建的 `sk_...` |
| 鉴权 | `Authorization: Bearer <client-api-key>` |
| 配置 | 值 |
| --- | --- |
| Base URL | `http://127.0.0.1:8080/v1` |
| API Key | 管理端创建的 `sk_...` |
| 鉴权 | `Authorization: Bearer <client-api-key>` |
```bash
curl http://127.0.0.1:8080/v1/models \
@@ -130,67 +130,70 @@ curl http://127.0.0.1:8080/v1/responses \
```
<details>
<summary>API 路由</summary>
<summary>公开数据面路由</summary>
| 路由 | 用途 |
| ---------------------------------- | ------------------------------ |
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
| `GET /v1/responses` | Responses WebSocket 升级 |
| `POST /v1/responses/review` | review 子代理请求 |
| `GET /v1/models` | 当前 Client Key 可用的模型列表 |
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
| 路由 | 用途 |
| --- | --- |
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
| `GET /v1/responses` | Responses WebSocket 升级 |
| `POST /v1/responses/review` | review 子代理请求 |
| `POST /v1/images/generations` | OpenAI Images 生成 JSON 透明转发 |
| `POST /v1/images/edits` | OpenAI Images 编辑 JSON 透明转发 |
| `GET /v1/models` | 当前 Client Key 账号范围内的模型列表 |
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
所有 `/v1/*` 路由都需要客户端 API Key。
所有 `/v1/*` 路由都需要 Client Key。Images 是 OpenAI Provider 自有端点,不读取模型字段,也不参与
文本模型映射。
</details>
完整的客户端与管理端路由、鉴权和 mutation 规则见 [接口文档](docs/api.md)。
完整的客户端与管理端路由、鉴权、请求字段和 mutation 语义见 [接口文档](docs/api.md)。
## 运维
## 运行维护
```bash
# 实例升级
# 同一大版本内升级发布镜像
docker compose -f deploy/compose.yaml pull codex-proxy-rs
docker compose -f deploy/compose.yaml up -d --no-build
# 日志
# 查看日志
docker compose -f deploy/compose.yaml logs -f codex-proxy-rs
# 从源码构建
# 从当前源码构建
docker compose -f deploy/compose.yaml build codex-proxy-rs
docker compose -f deploy/compose.yaml up -d
```
> [!IMPORTANT]
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点密钥和自更新状态。删除该目录会永久清除
> 运行状态。
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点和在线更新状态。删除该目录会永久清除
> 本地运行状态。
> [!WARNING]
> 不支持跨大版本在线升级。升级到新的大版本时,请使用全新的 `.runtime/` 数据目录重新部署,
> 并重新导入或重新授权 Provider 账号与客户端 Key。
> 不支持跨大版本在线升级。升级到新的大版本时请使用全新的 `.runtime/` 数据目录,并重新导入或授权
> Provider 账号与 Client Key。
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket upgrade 与真实客户端
IP;不要暴露 PostgreSQL 或 Redis。当前运行拓扑为单副本,不能通过复制应用容器扩容。
## 维护与发布
发布仓库版本必须从干净、已同步上游的发布分支运行:
仓库版本只能从干净、已同步上游的发布分支发布:
```bash
release/publish <version>
```
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
仓库发版和实例升级是三个独立状态。
脚本依赖已登录的 GitHub CLI(`gh auth login`),会更新 `release/version.yaml`、创建约定提交和带注释的
`v<version>` tag,原子推送分支与 tag,再触发 Release 构建。源码提交、仓库发版和运行实例升级是三个
独立状态;脚本不会登录服务器或改变任何运行实例。
## 文档
- [部署](deploy/README.md)
- [部署与运维](deploy/README.md)
- [客户端与管理端接口](docs/api.md)
- [架构与数据边界](docs/architecture.md)
- [系统架构](docs/architecture.md)
- [数据库迁移规则](backend/migrations/README.md)
- [Release](https://github.com/zyycn/codex-proxy-rs/releases)
- [容器镜像](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
+52 -78
View File
@@ -120,9 +120,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.91"
version = "0.1.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
dependencies = [
"proc-macro2",
"quote",
@@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.140.0"
version = "1.141.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9660cf991e512fbe6094f1041ff3d3282bc5aeeeb6184e8de437ec2de024a10"
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -451,9 +451,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.6.1"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8"
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
dependencies = [
"base64-simd",
"bytes",
@@ -578,9 +578,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64"
version = "0.23.0"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64-simd"
@@ -801,9 +801,9 @@ checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
[[package]]
name = "cookie"
version = "0.18.1"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
dependencies = [
"percent-encoding",
"time",
@@ -1246,7 +1246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -1384,9 +1384,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
dependencies = [
"futures-channel",
"futures-core",
@@ -1399,9 +1399,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
dependencies = [
"futures-core",
"futures-sink",
@@ -1409,15 +1409,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
[[package]]
name = "futures-executor"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
dependencies = [
"futures-core",
"futures-task",
@@ -1437,32 +1437,32 @@ dependencies = [
[[package]]
name = "futures-io"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
[[package]]
name = "futures-macro"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.3",
]
[[package]]
name = "futures-sink"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
[[package]]
name = "futures-task"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
[[package]]
name = "futures-timer"
@@ -1472,9 +1472,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
[[package]]
name = "futures-util"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
dependencies = [
"futures-channel",
"futures-core",
@@ -1493,7 +1493,7 @@ version = "0.0.0"
dependencies = [
"argon2",
"async-trait",
"base64 0.23.0",
"base64 0.23.1",
"chrono",
"chrono-tz",
"cron",
@@ -1519,7 +1519,7 @@ version = "0.1.0"
dependencies = [
"async-trait",
"axum",
"base64 0.23.0",
"base64 0.23.1",
"bytes",
"chrono",
"futures",
@@ -1686,9 +1686,9 @@ dependencies = [
[[package]]
name = "h2"
version = "0.4.15"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
dependencies = [
"atomic-waker",
"bytes",
@@ -2125,11 +2125,9 @@ dependencies = [
"base64 0.22.1",
"getrandom 0.2.17",
"js-sys",
"pem",
"serde",
"serde_json",
"signature 2.2.0",
"simple_asn1",
"zeroize",
]
@@ -2280,16 +2278,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-bigint"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.2"
@@ -2407,16 +2395,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
[[package]]
name = "pem"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
dependencies = [
"base64 0.22.1",
"serde_core",
]
[[package]]
name = "pem-rfc7468"
version = "1.0.0"
@@ -2597,12 +2575,13 @@ version = "0.0.0"
dependencies = [
"async-stream",
"async-trait",
"base64 0.23.0",
"base64 0.23.1",
"bytes",
"chrono",
"cookie",
"crypto_box",
"ed25519-dalek",
"flate2",
"futures",
"gateway-admin",
"gateway-core",
@@ -2610,7 +2589,6 @@ dependencies = [
"getrandom 0.4.3",
"hex",
"hyper-rustls",
"jsonwebtoken",
"reqwest",
"roxmltree",
"rustls",
@@ -2628,8 +2606,10 @@ dependencies = [
"tokio-tungstenite 0.28.0",
"tokio-util",
"tracing",
"tracing-subscriber",
"tungstenite 0.27.0",
"url",
"urlencoding",
"uuid",
"wiremock",
"zstd",
@@ -2641,7 +2621,7 @@ version = "0.0.0"
dependencies = [
"async-stream",
"async-trait",
"base64 0.23.0",
"base64 0.23.1",
"bytes",
"chrono",
"futures",
@@ -2734,7 +2714,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3004,7 +2984,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3323,18 +3303,6 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "simple_asn1"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
dependencies = [
"num-bigint",
"num-traits",
"thiserror",
"time",
]
[[package]]
name = "siphasher"
version = "1.0.3"
@@ -3660,23 +3628,23 @@ dependencies = [
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
name = "thiserror"
version = "2.0.19"
version = "2.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "2.0.19"
version = "2.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
dependencies = [
"proc-macro2",
"quote",
@@ -4112,6 +4080,12 @@ dependencies = [
"serde",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf-8"
version = "0.7.6"
+3 -2
View File
@@ -27,8 +27,9 @@ impl LoadableConfig for GatewayConfig {
return Err(ConfigError::InvalidField("schema_version"));
}
self.host.resolve_and_validate(source_dir)?;
let runtime_data_dir = self.host.runtime_data_dir().to_path_buf();
self.store
.resolve_and_validate(source_dir)
.resolve_and_validate(&runtime_data_dir)
.map_err(|_| ConfigError::InvalidField("store"))?;
self.admin
.resolve_and_validate(source_dir)
@@ -37,7 +38,7 @@ impl LoadableConfig for GatewayConfig {
.resolve_and_validate(source_dir)
.map_err(|_| ConfigError::InvalidField("api"))?;
self.openai
.resolve_and_validate(source_dir)
.resolve_and_validate(&runtime_data_dir)
.map_err(|_| ConfigError::InvalidField("openai"))?;
self.xai
.resolve_and_validate(source_dir)
+14 -3
View File
@@ -18,10 +18,18 @@ fn config_loader_should_load_complete_terminal_example() {
fn config_loader_should_resolve_paths_relative_to_config_file() {
let (config, _directory) = parse_config(&valid_config()).expect("resolved config");
let debug = format!("{config:?}");
assert!(debug.contains(".runtime/data"));
assert!(debug.contains(".runtime/logs"));
assert!(debug.contains("frontend/dist"));
}
#[test]
fn config_loader_should_reject_missing_runtime_data_dir() {
let config = valid_config().replace(" runtime_data_dir: '../.runtime/data'\n", "");
assert!(parse_config(&config).is_err());
}
#[test]
fn config_loader_should_inject_connection_passwords_into_urls() {
parse_config(&valid_config()).expect("Store validates password injection into both URLs");
@@ -134,8 +142,11 @@ fn config_loader_should_reject_removed_fingerprint_section() {
}
#[test]
fn config_loader_should_reject_invalid_codex_core_version() {
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
fn config_loader_should_reject_invalid_codex_cli_version() {
assert_rejected(valid_config().replace(
"codex_version: '0.147.0-alpha.6.6'",
"codex_version: 'latest'",
));
}
#[test]
@@ -154,7 +165,7 @@ fn config_loader_should_reject_zero_server_port() {
#[test]
fn config_loader_should_reject_invalid_desktop_profile_fields() {
assert_rejected(valid_config().replace("desktop_build: '6119'", "desktop_build: 'build'"));
assert_rejected(valid_config().replace("desktop_build: '6415'", "desktop_build: 'build'"));
}
fn assert_rejected(config: String) {
@@ -119,11 +119,11 @@ pub struct RetentionDecision {
/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。
///
/// 规则(`docs/s3-backup-design-audit.md` §4.4):
/// 自动保留规则:
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理。
/// - 自动计划备份满足任一启用阈值即可进入删除。
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)。
/// - 手动备份不受自动阈值影响;无 TTL,仅由管理员手动删除。
/// - 手动备份不进入本规则;手工/计划记录各自的 `expires_at` 由到期扫描处理。
#[must_use]
pub fn decide_retention(
retention_days: u32,
+59 -13
View File
@@ -220,7 +220,8 @@ impl BackupTask {
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
let Some(uploading_record) = self
.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
@@ -229,9 +230,12 @@ impl BackupTask {
now,
)
.await
.map_err(repo_error)?;
.map_err(repo_error)?
else {
return Ok(());
};
self.upload_and_verify(
record,
&uploading_record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
@@ -394,7 +398,8 @@ impl BackupTask {
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
let Some(uploading_record) = self
.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
@@ -403,10 +408,13 @@ impl BackupTask {
now,
)
.await
.map_err(repo_error)?;
.map_err(repo_error)?
else {
return Ok(());
};
self.upload_and_verify(
record,
&uploading_record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
@@ -448,8 +456,12 @@ impl BackupTask {
.await
{
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "上传到对象存储失败")
.await?;
self.fail_task(
record,
error.code(),
format!("上传到对象存储失败:{}", error.message()),
)
.await?;
return Ok(());
}
@@ -475,19 +487,46 @@ impl BackupTask {
);
}
}
Ok(_) => {
Ok(Some(remote)) => {
let size_matches = remote.size_bytes == size_bytes;
let sha256_matches = remote.sha256 == sha256;
warn!(
backup_id = %record.id,
expected_size_bytes = size_bytes,
remote_size_bytes = remote.size_bytes,
sha256_matches,
"备份远端对象校验不一致"
);
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(
record,
code::REMOTE_VERIFICATION_FAILED,
"远端对象大小或校验值不一致",
match (size_matches, sha256_matches) {
(false, true) => "远端对象大小不一致",
(true, false) => "远端对象 sha256 校验值不一致",
(false, false) => "远端对象大小和 sha256 校验值均不一致",
(true, true) => "远端对象校验状态异常",
},
)
.await?;
}
Ok(None) => {
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
self.fail_task(
record,
code::REMOTE_VERIFICATION_FAILED,
"上传完成后未找到远端对象",
)
.await?;
}
Err(error) => {
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "远端对象校验失败")
.await?;
self.fail_task(
record,
error.code(),
format!("远端对象校验失败:{}", error.message()),
)
.await?;
}
}
Ok(())
@@ -500,10 +539,17 @@ impl BackupTask {
error_code: &'static str,
message: impl Into<String>,
) -> Result<(), WorkerTaskError> {
let message = message.into();
warn!(
backup_id = %record.id,
error_code,
error_message = %message,
"备份任务失败"
);
let now = Utc::now();
let update = StatusTransitionUpdate {
error_code: Some(error_code.to_string()),
error_message: Some(message.into()),
error_message: Some(message),
completed_at: Some(now),
..Default::default()
};
+21 -9
View File
@@ -21,9 +21,9 @@ pub mod ports;
mod use_case;
pub use use_case::{
accounts::AccountsService, auth::AuthService, backup::BackupService,
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
settings::SettingsService, system::SystemService, xai::XaiService,
account_groups::AccountGroupService, accounts::AccountsService, auth::AuthService,
backup::BackupService, client_keys::ClientKeyService, observability::ObservabilityService,
openai::OpenAiService, settings::SettingsService, system::SystemService, xai::XaiService,
};
use model::{AdminError, AdminErrorKind};
@@ -33,10 +33,10 @@ use ports::{
system::SystemOperations,
};
use use_case::{
accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService,
client_keys::DefaultClientKeyService, observability::DefaultObservabilityService,
openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService,
xai::DefaultXaiService,
account_groups::DefaultAccountGroupService, accounts::DefaultAccountsService,
auth::DefaultAuthService, backup::DefaultBackupService, client_keys::DefaultClientKeyService,
observability::DefaultObservabilityService, openai::DefaultOpenAiService,
settings::DefaultSettingsService, system::DefaultSystemService, xai::DefaultXaiService,
};
const OPENAI_PROVIDER_KIND: &str = "openai";
@@ -147,6 +147,7 @@ pub enum AdminConfigError {
pub struct AdminServices {
auth: Arc<dyn AuthService>,
accounts: Arc<dyn AccountsService>,
account_groups: Arc<dyn AccountGroupService>,
client_keys: Arc<dyn ClientKeyService>,
observability: Arc<dyn ObservabilityService>,
settings: Arc<dyn SettingsService>,
@@ -167,6 +168,11 @@ impl AdminServices {
self.accounts.as_ref()
}
#[must_use]
pub fn account_groups(&self) -> &dyn AccountGroupService {
self.account_groups.as_ref()
}
#[must_use]
pub fn client_keys(&self) -> &dyn ClientKeyService {
self.client_keys.as_ref()
@@ -255,7 +261,6 @@ pub async fn initialize(
let accounts = Arc::new(DefaultAccountsService::new(
store.accounts(),
store.settings(),
registry.clone(),
snapshot.clone(),
probe.clone(),
@@ -275,6 +280,10 @@ pub async fn initialize(
let services = AdminServices {
auth,
accounts,
account_groups: Arc::new(DefaultAccountGroupService::new(
store.account_groups(),
snapshot.clone(),
)),
client_keys: Arc::new(DefaultClientKeyService::new(
store.client_keys(),
snapshot.clone(),
@@ -338,7 +347,10 @@ fn map_provider_registry_error(error: ProviderAdminError) -> AdminError {
}
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
ProviderAdminErrorKind::Unavailable | ProviderAdminErrorKind::CredentialRefreshRequired => {
AdminErrorKind::Unavailable
}
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
};
AdminError::new(kind, "Provider registry initialization failed")
@@ -0,0 +1,142 @@
//! Provider-neutral account group commands and projections.
use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use gateway_core::routing::AccountGroupId;
use super::{PageSize, Revision, observability::DecimalAmount};
/// Canonical `#RRGGBBAA` color persisted with an account group.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupColor(String);
impl AccountGroupColor {
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
(value.len() == 9
&& value.starts_with('#')
&& value[1..].bytes().all(|byte| byte.is_ascii_hexdigit()))
.then(|| Self(value.to_ascii_uppercase()))
}
}
/// Group member availability at the current observation time.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountGroupAccountSummary {
pub available: u64,
pub limited: u64,
pub total: u64,
}
/// Group scheduling slots derived from available accounts and runtime leases.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountGroupCapacity {
pub used_slots: Option<u64>,
pub total_slots: u64,
}
/// Successful, downstream-committed USD request costs for group accounts.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupUsage {
pub today_usd: DecimalAmount,
pub total_usd: DecimalAmount,
}
/// Lightweight group reference embedded in account and client-key views.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupRef {
pub id: AccountGroupId,
pub name: String,
pub color: AccountGroupColor,
pub enabled: bool,
}
/// Account group list query.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupListQuery {
pub page: u32,
pub page_size: PageSize,
pub search: Option<String>,
pub enabled: Option<bool>,
}
/// Complete account group summary.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupRecord {
pub id: AccountGroupId,
pub name: String,
pub description: Option<String>,
pub color: AccountGroupColor,
pub enabled: bool,
pub member_count: u64,
pub provider_counts: BTreeMap<String, u64>,
pub client_key_count: u64,
pub account_summary: AccountGroupAccountSummary,
pub capacity: AccountGroupCapacity,
pub usage: AccountGroupUsage,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
/// Paginated account groups.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupPage {
pub config_revision: Revision,
pub items: Vec<AccountGroupRecord>,
pub total: u64,
pub page: u32,
pub page_size: u16,
}
/// Create an account group.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreateAccountGroup {
pub name: String,
pub description: Option<String>,
pub color: AccountGroupColor,
}
/// Store-ready create command with a generated stable ID.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NewAccountGroup {
pub id: AccountGroupId,
pub name: String,
pub description: Option<String>,
pub color: AccountGroupColor,
}
/// Update an account group's descriptive fields.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdateAccountGroup {
pub id: AccountGroupId,
pub name: String,
pub description: Option<String>,
pub color: AccountGroupColor,
}
/// Enable or disable an account group.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SetAccountGroupEnabled {
pub id: AccountGroupId,
pub enabled: bool,
}
/// Delete an account group that is not referenced by a client key.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeleteAccountGroup {
pub id: AccountGroupId,
}
/// Result of any account-group mutation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupMutation {
pub config_revision: Revision,
pub id: AccountGroupId,
pub record: Option<AccountGroupRecord>,
}
@@ -1,159 +1,19 @@
//! 多 Provider 账号目录与连接测试的公共事实。
use std::{pin::Pin, time::SystemTime};
use std::pin::Pin;
use chrono::{DateTime, Utc};
use futures::Stream;
use gateway_core::routing::ProviderKind;
use super::{PageSize, Revision, observability::TimeRange};
use super::{PageSize, Revision, account_groups::AccountGroupRef, observability::TimeRange};
/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。
pub use gateway_core::engine::credential::AccountAvailability;
/// 管理页使用的归一化账号状态。
///
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountStatus {
/// 正常,可调度。
Normal,
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
QuotaExhausted,
/// 限流中(quota 快照级信号),自动恢复,无需干预。
RateLimited,
/// 管理员手动停用(`enabled = false`)。
Disabled,
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
Error,
}
impl AccountStatus {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Normal => "normal",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Disabled => "disabled",
Self::Error => "error",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"normal" => Some(Self::Normal),
"quota_exhausted" => Some(Self::QuotaExhausted),
"rate_limited" => Some(Self::RateLimited),
"disabled" => Some(Self::Disabled),
"error" => Some(Self::Error),
_ => None,
}
}
}
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusSignals {
pub enabled: bool,
pub availability: AccountAvailability,
pub access_token_expired: bool,
pub quota_limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
pub rate_limited_until: Option<SystemTime>,
}
impl AccountStatusSignals {
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
#[must_use]
pub const fn derive(self) -> AccountStatus {
if !self.enabled {
AccountStatus::Disabled
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
AccountStatus::QuotaExhausted
} else if self.access_token_expired
|| matches!(
self.availability,
AccountAvailability::Expired
| AccountAvailability::Invalid
| AccountAvailability::Banned
| AccountAvailability::Unknown
)
{
AccountStatus::Error
} else if self.rate_limited_until.is_some() {
AccountStatus::RateLimited
} else if self.quota_limit_reached {
AccountStatus::QuotaExhausted
} else {
AccountStatus::Normal
}
}
}
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
///
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountErrorReason {
Expired,
TokenExpired,
Invalid,
Banned,
Unknown,
}
impl AccountErrorReason {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Expired => "expired",
Self::TokenExpired => "token_expired",
Self::Invalid => "invalid",
Self::Banned => "banned",
Self::Unknown => "unknown",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"expired" => Some(Self::Expired),
"token_expired" => Some(Self::TokenExpired),
"invalid" => Some(Self::Invalid),
"banned" => Some(Self::Banned),
"unknown" => Some(Self::Unknown),
_ => None,
}
}
}
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
///
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
#[must_use]
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
if !signals.enabled {
return None;
}
match signals.availability {
AccountAvailability::QuotaExhausted => None,
AccountAvailability::Ready => signals
.access_token_expired
.then_some(AccountErrorReason::TokenExpired),
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
}
}
pub use gateway_core::engine::credential::{
AccountConcurrencyLimit, AccountErrorReason, AccountStatus, AccountStatusFacts,
AccountStatusProjection, AccountWeight, CredentialState, QuotaAccessState, QuotaEvidence,
QuotaState, resolve_account_status,
};
/// 账号列表排序字段。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -186,19 +46,28 @@ pub struct AccountListQuery {
pub page: u32,
pub page_size: PageSize,
pub provider_kind: Option<ProviderKind>,
pub group_filter: Option<AccountGroupFilter>,
pub search: Option<String>,
pub status: Option<AccountStatus>,
pub sort: Option<AccountSort>,
}
/// Optional account membership filter.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AccountGroupFilter {
Group(gateway_core::routing::AccountGroupId),
Ungrouped,
}
/// 账号公共存储投影;Provider 专属字段不进入此结构。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountRecord {
pub id: String,
pub provider_kind: ProviderKind,
pub groups: Vec<AccountGroupRef>,
pub name: String,
pub email: Option<String>,
pub upstream_user_id: String,
pub upstream_user_id: Option<String>,
pub upstream_account_id: Option<String>,
pub plan_type: Option<String>,
pub authentication_kind: String,
@@ -207,11 +76,13 @@ pub struct AccountRecord {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub concurrency_limit: Option<AccountConcurrencyLimit>,
pub weight: AccountWeight,
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
pub quota: QuotaState,
pub last_error_reason: Option<AccountErrorReason>,
pub last_error_message: Option<String>,
pub quota_observed_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -296,11 +167,18 @@ pub struct AccountUsageWindowResult {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountPage {
pub config_revision: Revision,
pub items: Vec<AccountRecord>,
pub items: Vec<AccountPageItem>,
pub total: u64,
pub summary: AccountSummary,
}
/// 同一状态快照下的账号事实与唯一状态投影。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountPageItem {
pub account: AccountRecord,
pub projection: AccountStatusProjection,
}
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
@@ -314,11 +192,38 @@ pub struct AccountSummary {
pub error: u64,
}
/// 账号启停写入命令。
/// 账号可编辑事实的一次性替换命令。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SetAccountEnabled {
pub struct UpdateAccount {
pub account_id: String,
pub enabled: bool,
pub concurrency_limit: Option<AccountConcurrencyLimit>,
pub weight: AccountWeight,
pub group_ids: Vec<gateway_core::routing::AccountGroupId>,
}
/// 账号更新结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountUpdateResult {
pub config_revision: Revision,
pub account_id: gateway_core::engine::credential::ProviderAccountId,
}
/// 一批账号可编辑事实的一次性替换命令。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BatchUpdateAccounts {
pub account_ids: Vec<String>,
pub enabled: bool,
pub concurrency_limit: Option<AccountConcurrencyLimit>,
pub weight: AccountWeight,
pub group_ids: Vec<gateway_core::routing::AccountGroupId>,
}
/// 批量账号更新结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountsUpdateResult {
pub config_revision: Revision,
pub account_ids: Vec<gateway_core::engine::credential::ProviderAccountId>,
}
/// 账号批量删除命令。
@@ -342,9 +247,7 @@ pub enum AccountConnectionTestEvent {
Content {
text: String,
},
Completed {
account_status: AccountStatus,
},
Completed,
Failed {
message: String,
provider_error_code: Option<String>,
@@ -352,7 +255,6 @@ pub enum AccountConnectionTestEvent {
upstream_status: Option<u16>,
upstream_content_type: Option<String>,
upstream_body: Option<String>,
account_status: AccountStatus,
},
}
@@ -288,7 +288,7 @@ pub struct BackupRecord {
pub error_message: Option<String>,
pub started_at: Option<DateTime<Utc>>,
pub completed_at: Option<DateTime<Utc>>,
/// 手动过期时间;创建时确定,到期自动清理。
/// 创建时确定的过期时间;手工和计划备份均可设置,到期自动清理。
pub expires_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
@@ -6,10 +6,10 @@ use chrono::{DateTime, Utc};
use gateway_core::{
policy::{ClientApiKeyId, RateLimits},
routing::ProviderKind,
routing::{AccountGroupId, ProviderKind},
};
use super::{AdminModelError, Revision};
use super::{AdminModelError, Revision, account_groups::AccountGroupRef};
/// Client Key 列表保持旧 HTTP 合同允许的完整非零 `u16` 页大小。
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -88,7 +88,8 @@ pub struct ClientKeyRecord {
pub id: ClientApiKeyId,
pub name: String,
pub label: Option<String>,
pub provider_kind: ProviderKind,
pub groups: Vec<AccountGroupRef>,
pub provider_kinds: Vec<ProviderKind>,
pub prefix: String,
pub enabled: bool,
pub limits: RateLimits,
@@ -143,7 +144,7 @@ impl fmt::Debug for ClientKeySecret {
pub struct CreateClientKey {
pub name: String,
pub label: Option<String>,
pub provider_kind: ProviderKind,
pub group_ids: Vec<AccountGroupId>,
pub limits: RateLimits,
}
@@ -153,7 +154,7 @@ pub struct NewClientKey {
pub id: ClientApiKeyId,
pub name: String,
pub label: Option<String>,
pub provider_kind: ProviderKind,
pub group_ids: Vec<AccountGroupId>,
pub limits: RateLimits,
pub plaintext: String,
}
@@ -164,7 +165,7 @@ impl fmt::Debug for NewClientKey {
.debug_struct("NewClientKey")
.field("id", &self.id)
.field("name", &self.name)
.field("provider_kind", &self.provider_kind)
.field("group_ids", &self.group_ids)
.field("plaintext", &"[REDACTED]")
.finish_non_exhaustive()
}
@@ -176,7 +177,7 @@ pub struct UpdateClientKey {
pub id: ClientApiKeyId,
pub name: String,
pub label: Option<String>,
pub provider_kind: ProviderKind,
pub group_ids: Vec<AccountGroupId>,
pub limits: RateLimits,
}
@@ -2,6 +2,7 @@
use std::num::{NonZeroU16, NonZeroU64};
pub mod account_groups;
pub mod accounts;
pub mod auth;
pub mod backup;
@@ -439,25 +439,15 @@ pub struct RequestMetricPoint {
pub costs: Vec<CurrencyCost>,
}
/// 账号池容量统计。
///
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
///
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
/// 以便状态详情展示和诊断。
/// 账号池的统一五态统计。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountPoolMetrics {
pub total: u64,
pub enabled: u64,
pub unavailable: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub rate_limited: u64,
pub disabled: u64,
pub banned: u64,
pub error: u64,
}
/// Dashboard 中一个账号的模型级用量事实。
@@ -488,7 +478,7 @@ pub struct DashboardAccountRequestBucket {
pub request_count: u64,
}
/// Dashboard 中一个账号的完整用量与公共状态事实。
/// Dashboard 中一个账号的完整用量事实。
#[derive(Debug, Clone, PartialEq)]
pub struct DashboardAccountUsage {
pub account_id: String,
@@ -497,8 +487,6 @@ pub struct DashboardAccountUsage {
pub name: String,
pub email: Option<String>,
pub plan_type: Option<String>,
pub enabled: bool,
pub availability: String,
pub request_count: u64,
pub success_count: u64,
pub input_tokens: Option<u64>,
@@ -525,18 +513,31 @@ pub struct DashboardAccountUsage {
/// Dashboard 当前账号池的可重建运行时槽位事实。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DashboardRuntimeSlots {
/// 与持久账号池指标相同谓词下可参与调度的账号数。
pub active_accounts: u64,
/// 继承运行参数默认并发上限的可调度账号数。
pub inherited_accounts: u64,
/// 已配置账号级并发上限的可调度账号槽位总数。
pub overridden_slots: u64,
/// Redis 可用时的实时 in-flight 槽位总数;不可用时为 `None`。
pub used_slots: Option<u64>,
}
/// 仪表盘所需的公共观测事实。
/// 仪表盘卡片脚注展示的全历史累计。
#[derive(Debug, Clone, Default, PartialEq)]
pub struct DashboardTotals {
pub request_count: u64,
pub input_tokens: u64,
pub cached_tokens: u64,
pub total_tokens: u64,
pub billing_usd: Option<DecimalAmount>,
}
/// 仪表盘所需的当前区间事实及全历史累计。
#[derive(Debug, Clone, PartialEq)]
pub struct DashboardObservation {
pub range: TimeRange,
pub requests: RequestMetrics,
pub attempts: AttemptMetrics,
pub totals: DashboardTotals,
pub provider_accounts: AccountPoolMetrics,
pub trend: Vec<RequestMetricPoint>,
pub account_usage: Vec<DashboardAccountUsage>,
@@ -549,11 +550,14 @@ pub struct UsageRecord {
pub id: String,
pub client_api_key_ref: String,
pub config_revision: u64,
pub routing_scope: String,
pub routing_group_refs: Vec<String>,
pub routing_group_names_snapshot: Vec<String>,
pub protocol: String,
pub operation: String,
pub endpoint: String,
pub client_transport: String,
pub requested_model_id: String,
pub requested_model_id: Option<String>,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
@@ -8,13 +8,11 @@ use gateway_core::{
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
routing::{ProviderKind, UpstreamModelId},
};
use uuid::Uuid;
use super::{
AdminError, MutationActor, MutationContext, PageSize, Revision,
accounts::{
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
AccountUsage,
},
accounts::{AccountRecord, AccountSummary, AccountUsage, CredentialState},
};
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
@@ -55,24 +53,24 @@ pub struct CredentialCursor {
/// Provider credential 列表查询。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CredentialListQuery {
pub availability: Option<CredentialAvailabilityFilter>,
pub credential_state: Option<CredentialStateFilter>,
pub enabled: Option<bool>,
pub window: CredentialListWindow,
}
/// Credential 可用性筛选;支持一个 wire 值对应多个持久状态。
/// Credential 状态筛选。
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CredentialAvailabilityFilter {
Exact(AccountAvailability),
AnyOf(Vec<AccountAvailability>),
pub enum CredentialStateFilter {
Exact(CredentialState),
AnyOf(Vec<CredentialState>),
}
impl CredentialAvailabilityFilter {
impl CredentialStateFilter {
#[must_use]
pub fn matches(&self, availability: AccountAvailability) -> bool {
pub fn matches(&self, credential_state: CredentialState) -> bool {
match self {
Self::Exact(expected) => *expected == availability,
Self::AnyOf(expected) => expected.contains(&availability),
Self::Exact(expected) => *expected == credential_state,
Self::AnyOf(expected) => expected.contains(&credential_state),
}
}
}
@@ -146,7 +144,7 @@ pub struct PreparedCredentialCreate {
pub provider_kind: ProviderKind,
pub name: String,
pub email: Option<String>,
pub upstream_user_id: String,
pub upstream_user_id: Option<String>,
pub upstream_account_id: Option<String>,
pub plan_type: Option<String>,
pub authentication_kind: String,
@@ -155,8 +153,8 @@ pub struct PreparedCredentialCreate {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
}
/// Provider 对一份导入文档的完整验证结果。
@@ -564,13 +562,44 @@ impl fmt::Debug for RotateCredential {
}
}
/// 通用账号用量能否可靠归属到一个 Provider quota 窗口。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuotaLocalUsageAttribution {
/// 窗口覆盖账号的全部请求,可按账号与时间范围聚合。
AccountWide,
/// 窗口需要 Provider / 模型级归属,通用账号聚合不可用。
Unavailable,
}
/// Provider quota bucket 中窗口的官方位置语义。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProviderQuotaWindowRole {
Primary,
Secondary,
Monthly,
}
impl ProviderQuotaWindowRole {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Primary => "primary",
Self::Secondary => "secondary",
Self::Monthly => "monthly",
}
}
}
/// 一个 Provider quota 窗口的公共投影。
#[derive(Debug, Clone, PartialEq)]
pub struct ProviderQuotaWindow {
pub key: String,
pub group: String,
pub label: String,
pub source: Option<String>,
pub limit_id: Option<String>,
pub limit_name: Option<String>,
pub role: Option<ProviderQuotaWindowRole>,
pub local_usage_attribution: QuotaLocalUsageAttribution,
pub window_seconds: Option<u64>,
pub used_percent: Option<f64>,
pub reset_at: Option<DateTime<Utc>>,
@@ -593,50 +622,153 @@ pub struct ProviderQuota {
pub observed_at: Option<DateTime<Utc>>,
pub refresh_token_expires_at: Option<DateTime<Utc>>,
pub windows: Vec<ProviderQuotaWindow>,
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
/// 展示用快照级触顶事实(顶层或任一窗口触顶);不参与账号五态派生。
pub limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
pub rate_limited_until: Option<DateTime<Utc>>,
pub provider_data: Option<ProviderDocument>,
}
/// Provider 返回的一张安全主动额度重置卡。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderResetCredit {
pub id: String,
pub status: Option<String>,
pub title: Option<String>,
pub expires_at: Option<DateTime<Utc>>,
pub reset_type: Option<String>,
}
/// Provider 主动额度重置卡列表。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderResetCredits {
pub available_count: u64,
pub credits: Vec<ProviderResetCredit>,
}
/// 一次主动额度重置卡消费命令。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConsumeProviderResetCredit {
pub account_id: ProviderAccountId,
pub credit_id: Option<String>,
pub redeem_request_id: Uuid,
}
/// Provider 返回的主动额度重置卡消费结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderResetCreditResult {
pub code: String,
pub credit: Option<ProviderResetCredit>,
}
impl ProviderQuota {
/// 返回 Dashboard 使用的代表性额度比例。
/// 返回账号展开面板使用的当前额度窗口用量。
///
/// 优先使用短周期窗口,再依次使用周、月和其它窗口;同一优先级取较高的已用比例。
/// 这里只解释跨 Provider 共享的窗口语义,绝不读取 Provider 私有 JSON。
/// Provider 已提供账号级本地用量时,固定窗口和无固定重置点的滚动窗口
/// 都可作为代表用量;优先级与 Dashboard 代表性额度窗口一致,
/// 同一优先级保持 Provider 投影顺序。
#[must_use]
pub fn representative_used_percent(&self) -> Option<f64> {
pub fn representative_window_usage(&self) -> Option<&AccountUsage> {
self.windows
.iter()
.enumerate()
.filter(|(_, window)| {
window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
&& window.window_seconds.is_some()
&& window.local_usage.is_some()
})
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
.and_then(|(_, window)| window.local_usage.as_ref())
}
/// 返回 Dashboard 使用的代表性额度比例。
///
/// 优先使用覆盖账号全部请求的窗口;同一归属范围内再依次使用短周期、周、月
/// 和其它窗口,同一优先级取较高的已用比例。这里只解释跨 Provider 共享的窗口
/// 语义,绝不读取 Provider 私有 JSON。
#[must_use]
pub fn representative_used_percent(&self) -> Option<f64> {
if self.limit_reached {
return Some(100.0);
}
self.representative_used_window()
.map(|(_, used_percent)| used_percent)
}
/// 将已确认的账号级额度耗尽事实投影到展示窗口。
///
/// Provider 已指出具体触顶窗口时只归一化这些窗口;否则归一化 Dashboard
/// 同样会选择的代表窗口,避免把多个独立额度窗口全部伪造成已用尽。
pub fn apply_limit_reached_display(&mut self) {
if !self.limit_reached {
return;
}
let reached_windows = self
.windows
.iter()
.enumerate()
.filter_map(|(index, window)| window.limit_reached.then_some(index))
.collect::<Vec<_>>();
if !reached_windows.is_empty() {
for index in reached_windows {
self.windows[index].used_percent = Some(100.0);
}
return;
}
let representative = self
.representative_used_window()
.map(|(index, _)| index)
.or_else(|| {
self.windows
.iter()
.enumerate()
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
.map(|(index, _)| index)
});
if let Some(index) = representative {
self.windows[index].used_percent = Some(100.0);
self.windows[index].limit_reached = true;
}
}
fn representative_used_window(&self) -> Option<(usize, f64)> {
self.windows
.iter()
.enumerate()
.filter_map(|window| {
let (index, window) = window;
let used_percent = window
.used_percent
.filter(|value| value.is_finite())
.map(|value| value.clamp(0.0, 100.0))?;
Some((quota_usage_priority(window), used_percent))
Some((index, quota_usage_priority(window), used_percent))
})
.fold(None::<(u8, f64)>, |selected, candidate| match selected {
Some(current)
if current.0 < candidate.0
|| (current.0 == candidate.0 && current.1 >= candidate.1) =>
{
.fold(
None::<(usize, (u8, u8), f64)>,
|selected, candidate| match selected {
Some(current)
}
_ => Some(candidate),
})
.map(|(_, used_percent)| used_percent)
if current.1 < candidate.1
|| (current.1 == candidate.1 && current.2 >= candidate.2) =>
{
Some(current)
}
_ => Some(candidate),
},
)
.map(|(index, _, used_percent)| (index, used_percent))
}
}
fn quota_usage_priority(window: &ProviderQuotaWindow) -> u8 {
match window.group.as_str() {
fn quota_usage_priority(window: &ProviderQuotaWindow) -> (u8, u8) {
let attribution = match window.local_usage_attribution {
QuotaLocalUsageAttribution::AccountWide => 0,
QuotaLocalUsageAttribution::Unavailable => 1,
};
let duration = match window.group.as_str() {
"shortTerm" if window.window_seconds.is_some_and(is_week_window) => 1,
"shortTerm" => 0,
"monthly" => 2,
_ => 3,
}
};
(attribution, duration)
}
fn is_week_window(seconds: u64) -> bool {
@@ -693,9 +825,7 @@ impl fmt::Debug for ProviderExport {
#[derive(Debug, Clone, PartialEq)]
pub struct AccountDirectoryItem {
pub account: AccountRecord,
pub status: AccountStatus,
/// `status == Error` 时的具体原因;其余状态为 `None`。
pub error_reason: Option<AccountErrorReason>,
pub projection: gateway_core::engine::credential::AccountStatusProjection,
pub usage: Option<super::accounts::AccountUsage>,
pub quota: ProviderQuota,
}
@@ -13,11 +13,11 @@ use crate::model::observability::{
CalculatedBillingBreakdown, DashboardWireProfile, ProviderBillingInput,
};
use crate::model::provider_credentials::{
AuthorizationStarted, CompleteAuthorization, PendingAuthorizationMutation,
PrepareCredentialImport, PrepareCredentialRefresh, PrepareCredentialRotation,
PreparedAuthorizationCommit, PreparedCredentialImport, PreparedCredentialRotation,
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
ProviderQuotaRequest,
AuthorizationStarted, CompleteAuthorization, ConsumeProviderResetCredit,
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
PrepareCredentialRotation, PreparedAuthorizationCommit, PreparedCredentialImport,
PreparedCredentialRotation, ProviderExport, ProviderExportCredentialInput, ProviderModels,
ProviderQuota, ProviderQuotaRequest, ProviderResetCreditResult, ProviderResetCredits,
};
/// Provider 管理失败的稳定分类。
@@ -28,26 +28,56 @@ pub enum ProviderAdminErrorKind {
NotFound,
Conflict,
Unavailable,
CredentialRefreshRequired,
BadGateway,
Internal,
}
/// 不泄漏 OAuth material 与 Provider 响应体的管理错误。
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
/// 不携带 OAuth 请求材料的管理错误。
///
/// `message` 用于当前认证管理请求展示;上游没有标准 `error.message` 时,
/// Provider 可以按其官方契约回退为完整的非成功响应正文。
#[derive(Clone, PartialEq, Eq, thiserror::Error)]
#[error("provider admin operation failed: {kind:?}")]
pub struct ProviderAdminError {
kind: ProviderAdminErrorKind,
message: Option<String>,
}
impl std::fmt::Debug for ProviderAdminError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ProviderAdminError")
.field("kind", &self.kind)
.field("message", &self.message.as_ref().map(|_| "<redacted>"))
.finish()
}
}
impl ProviderAdminError {
#[must_use]
pub const fn new(kind: ProviderAdminErrorKind) -> Self {
Self { kind }
Self {
kind,
message: None,
}
}
#[must_use]
pub fn with_message(mut self, message: impl Into<String>) -> Self {
self.message = Some(message.into());
self
}
#[must_use]
pub const fn kind(&self) -> ProviderAdminErrorKind {
self.kind
}
#[must_use]
pub fn message(&self) -> Option<&str> {
self.message.as_deref()
}
}
/// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力。
@@ -116,6 +146,22 @@ pub trait ProviderAdmin: Send + Sync {
request: ProviderQuotaRequest,
) -> Result<ProviderQuota, ProviderAdminError>;
/// 查询 Provider 主动额度重置卡;不支持该能力的 Provider 使用默认拒绝。
async fn reset_credits(
&self,
_account_id: &ProviderAccountId,
) -> Result<ProviderResetCredits, ProviderAdminError> {
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
}
/// 消费 Provider 主动额度重置卡;不支持该能力的 Provider 使用默认拒绝。
async fn consume_reset_credit(
&self,
_command: ConsumeProviderResetCredit,
) -> Result<ProviderResetCreditResult, ProviderAdminError> {
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
}
async fn models(
&self,
account_id: &ProviderAccountId,
@@ -10,9 +10,14 @@ use chrono::{DateTime, Utc};
use super::backup::BackupStorePorts;
use crate::model::{
MutationContext, Revision,
account_groups::{
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, DeleteAccountGroup,
NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
},
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
AccountListQuery, AccountPage, AccountPageItem, AccountUpdateResult, AccountUsage,
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
},
auth::{AdminAuditEvent, AdminSession},
client_keys::{
@@ -83,7 +88,7 @@ pub type AdminStoreResult<T> = Result<T, AdminStoreError>;
pub trait AccountStore: Send + Sync {
async fn list_accounts(&self, query: AccountListQuery) -> AdminStoreResult<AccountPage>;
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>>;
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>>;
async fn load_account_usage(
&self,
@@ -138,11 +143,23 @@ pub trait AccountStore: Send + Sync {
context: &MutationContext,
) -> AdminStoreResult<CredentialMutationResult>;
async fn set_account_enabled(
async fn update_account(
&self,
command: SetAccountEnabled,
command: UpdateAccount,
context: &MutationContext,
) -> AdminStoreResult<Revision>;
) -> AdminStoreResult<AccountUpdateResult>;
async fn recover_account(
&self,
account_id: &gateway_core::engine::credential::ProviderAccountId,
context: &MutationContext,
) -> AdminStoreResult<AccountUpdateResult>;
async fn batch_update_accounts(
&self,
command: BatchUpdateAccounts,
context: &MutationContext,
) -> AdminStoreResult<AccountsUpdateResult>;
async fn delete_accounts(
&self,
@@ -215,10 +232,48 @@ pub trait ClientKeyStore: Send + Sync {
) -> AdminStoreResult<Revision>;
}
/// Provider-neutral account group management transactions.
#[async_trait]
pub trait AccountGroupStore: Send + Sync {
async fn list_account_groups(
&self,
query: AccountGroupListQuery,
) -> AdminStoreResult<AccountGroupPage>;
async fn create_account_group(
&self,
command: NewAccountGroup,
context: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation>;
async fn update_account_group(
&self,
command: UpdateAccountGroup,
context: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation>;
async fn set_account_group_enabled(
&self,
command: SetAccountGroupEnabled,
context: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation>;
async fn delete_account_group(
&self,
command: DeleteAccountGroup,
context: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation>;
}
/// 用量、趋势、诊断与运维错误的只读能力。
#[async_trait]
pub trait ObservabilityStore: Send + Sync {
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation>;
/// 返回历史统计区间和指定观测时刻下的实时账号状态。
async fn dashboard_summary(
&self,
range: TimeRange,
observed_at: DateTime<Utc>,
) -> AdminStoreResult<DashboardObservation>;
/// 返回 Dashboard 可选的实时槽位事实。
///
@@ -297,6 +352,7 @@ pub trait SettingsStore: Send + Sync {
#[derive(Clone)]
pub struct AdminStorePorts {
accounts: Arc<dyn AccountStore>,
account_groups: Arc<dyn AccountGroupStore>,
auth: Arc<dyn AuthStore>,
client_keys: Arc<dyn ClientKeyStore>,
observability: Arc<dyn ObservabilityStore>,
@@ -308,6 +364,7 @@ impl AdminStorePorts {
#[must_use]
pub fn new(
accounts: Arc<dyn AccountStore>,
account_groups: Arc<dyn AccountGroupStore>,
auth: Arc<dyn AuthStore>,
client_keys: Arc<dyn ClientKeyStore>,
observability: Arc<dyn ObservabilityStore>,
@@ -316,6 +373,7 @@ impl AdminStorePorts {
) -> Self {
Self {
accounts,
account_groups,
auth,
client_keys,
observability,
@@ -329,6 +387,11 @@ impl AdminStorePorts {
self.accounts.clone()
}
#[must_use]
pub fn account_groups(&self) -> Arc<dyn AccountGroupStore> {
self.account_groups.clone()
}
#[must_use]
pub fn auth(&self) -> Arc<dyn AuthStore> {
self.auth.clone()
@@ -0,0 +1,130 @@
//! Account group management use cases.
use std::sync::Arc;
use async_trait::async_trait;
use gateway_core::routing::{AccountGroupId, snapshot::SnapshotControl};
use uuid::Uuid;
use crate::{
model::{
AdminError, MutationContext,
account_groups::{
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, CreateAccountGroup,
DeleteAccountGroup, NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
},
},
ports::store::AccountGroupStore,
};
use super::{map_store_error, publish_committed};
/// API-facing account group management service.
#[async_trait]
pub trait AccountGroupService: Send + Sync {
async fn list(&self, query: AccountGroupListQuery) -> Result<AccountGroupPage, AdminError>;
async fn create(
&self,
context: &MutationContext,
command: CreateAccountGroup,
) -> Result<AccountGroupMutation, AdminError>;
async fn update(
&self,
context: &MutationContext,
command: UpdateAccountGroup,
) -> Result<AccountGroupMutation, AdminError>;
async fn set_enabled(
&self,
context: &MutationContext,
command: SetAccountGroupEnabled,
) -> Result<AccountGroupMutation, AdminError>;
async fn delete(
&self,
context: &MutationContext,
command: DeleteAccountGroup,
) -> Result<AccountGroupMutation, AdminError>;
}
pub(crate) struct DefaultAccountGroupService {
store: Arc<dyn AccountGroupStore>,
snapshot: Arc<dyn SnapshotControl>,
}
impl DefaultAccountGroupService {
#[must_use]
pub(crate) fn new(
store: Arc<dyn AccountGroupStore>,
snapshot: Arc<dyn SnapshotControl>,
) -> Self {
Self { store, snapshot }
}
async fn publish(
&self,
result: Result<AccountGroupMutation, crate::ports::store::AdminStoreError>,
) -> Result<AccountGroupMutation, AdminError> {
let result = result.map_err(|error| map_store_error(error, "account group"))?;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
}
#[async_trait]
impl AccountGroupService for DefaultAccountGroupService {
async fn list(&self, query: AccountGroupListQuery) -> Result<AccountGroupPage, AdminError> {
self.store
.list_account_groups(query)
.await
.map_err(|error| map_store_error(error, "account group"))
}
async fn create(
&self,
context: &MutationContext,
command: CreateAccountGroup,
) -> Result<AccountGroupMutation, AdminError> {
let id = AccountGroupId::new(format!("grp_{}", Uuid::now_v7().simple()))
.map_err(|_| AdminError::internal("Failed to create account group ID"))?;
self.publish(
self.store
.create_account_group(
NewAccountGroup {
id,
name: command.name,
description: command.description,
color: command.color,
},
context,
)
.await,
)
.await
}
async fn update(
&self,
context: &MutationContext,
command: UpdateAccountGroup,
) -> Result<AccountGroupMutation, AdminError> {
self.publish(self.store.update_account_group(command, context).await)
.await
}
async fn set_enabled(
&self,
context: &MutationContext,
command: SetAccountGroupEnabled,
) -> Result<AccountGroupMutation, AdminError> {
self.publish(self.store.set_account_group_enabled(command, context).await)
.await
}
async fn delete(
&self,
context: &MutationContext,
command: DeleteAccountGroup,
) -> Result<AccountGroupMutation, AdminError> {
self.publish(self.store.delete_account_group(command, context).await)
.await
}
}
@@ -1,6 +1,6 @@
//! 统一账号目录与跨 Provider 动态分派。
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
use std::{collections::BTreeMap, sync::Arc};
use async_trait::async_trait;
use chrono::{Duration, Utc};
@@ -18,20 +18,18 @@ use crate::{
AdminError, MutationContext,
accounts::{
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
derive_error_reason,
AccountPageItem, AccountUpdateResult, AccountUsageWindowQuery, AccountsUpdateResult,
BatchUpdateAccounts, UpdateAccount,
},
observability::TimeRange,
provider_credentials::{
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
PrepareCredentialRefresh, ProviderModels, ProviderQuota, ProviderQuotaRequest,
ProviderQuotaWindow,
ConsumeProviderResetCredit, PrepareCredentialRefresh, ProviderModels, ProviderQuota,
ProviderQuotaRequest, ProviderQuotaWindow, ProviderResetCreditResult,
ProviderResetCredits, QuotaLocalUsageAttribution,
},
},
ports::{
provider::ProviderAdminRegistry,
store::{AccountStore, SettingsStore},
},
ports::{provider::ProviderAdminRegistry, store::AccountStore},
};
use super::{
@@ -58,12 +56,50 @@ pub trait AccountsService: Send + Sync {
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError>;
async fn recover(
&self,
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError>;
async fn update(
&self,
context: &MutationContext,
command: UpdateAccount,
) -> Result<AccountUpdateResult, AdminError>;
async fn batch_update(
&self,
context: &MutationContext,
command: BatchUpdateAccounts,
) -> Result<AccountsUpdateResult, AdminError>;
async fn quota(
&self,
account_id: &ProviderAccountId,
refresh: bool,
) -> Result<AccountDirectoryItem, AdminError>;
async fn reset_credits(
&self,
_context: &MutationContext,
_account_id: ProviderAccountId,
) -> Result<ProviderResetCredits, AdminError> {
Err(AdminError::invalid(
"Provider reset credits are not supported",
))
}
async fn consume_reset_credit(
&self,
_context: &MutationContext,
_command: ConsumeProviderResetCredit,
) -> Result<ProviderResetCreditResult, AdminError> {
Err(AdminError::invalid(
"Provider reset credits are not supported",
))
}
async fn models(
&self,
account_id: &ProviderAccountId,
@@ -79,34 +115,46 @@ pub trait AccountsService: Send + Sync {
pub(crate) struct DefaultAccountsService {
accounts: Arc<dyn AccountStore>,
settings: Arc<dyn SettingsStore>,
providers: ProviderAdminRegistry,
snapshot: Arc<dyn SnapshotControl>,
probe: Arc<dyn AccountProbe>,
reset_credit_locks:
Arc<futures::lock::Mutex<BTreeMap<ProviderAccountId, Arc<futures::lock::Mutex<()>>>>>,
}
impl DefaultAccountsService {
#[must_use]
pub(crate) const fn new(
pub(crate) fn new(
accounts: Arc<dyn AccountStore>,
settings: Arc<dyn SettingsStore>,
providers: ProviderAdminRegistry,
snapshot: Arc<dyn SnapshotControl>,
probe: Arc<dyn AccountProbe>,
) -> Self {
Self {
accounts,
settings,
providers,
snapshot,
probe,
reset_credit_locks: Arc::new(futures::lock::Mutex::new(BTreeMap::new())),
}
}
async fn reset_credit_lock(
&self,
account_id: &ProviderAccountId,
) -> Arc<futures::lock::Mutex<()>> {
let mut locks = self.reset_credit_locks.lock().await;
Arc::clone(
locks
.entry(account_id.clone())
.or_insert_with(|| Arc::new(futures::lock::Mutex::new(()))),
)
}
async fn load_account(
&self,
account_id: &ProviderAccountId,
) -> Result<AccountRecord, AdminError> {
) -> Result<AccountPageItem, AdminError> {
self.accounts
.load_account(account_id.as_str())
.await
@@ -119,32 +167,32 @@ impl DefaultAccountsService {
account_id: &ProviderAccountId,
) -> Result<
(
AccountRecord,
AccountPageItem,
Arc<dyn crate::ports::provider::ProviderAdmin>,
),
AdminError,
> {
let account = self.load_account(account_id).await?;
let item = self.load_account(account_id).await?;
let provider = self
.providers
.require(&account.provider_kind)
.require(&item.account.provider_kind)
.map_err(|error| map_provider_error(error, "provider account"))?;
Ok((account, provider))
Ok((item, provider))
}
async fn attach_quota_local_usage(
&self,
accounts: &[AccountRecord],
accounts: &[AccountPageItem],
quotas: &mut [ProviderQuota],
) -> Result<(), AdminError> {
let windows = accounts
.iter()
.zip(quotas.iter())
.flat_map(|(account, quota)| {
.flat_map(|(item, quota)| {
quota
.windows
.iter()
.filter_map(|window| quota_usage_window(&account.id, window))
.filter_map(|window| quota_usage_window(&item.account.id, window))
})
.collect::<Vec<_>>();
if windows.is_empty() {
@@ -158,10 +206,12 @@ impl DefaultAccountsService {
.into_iter()
.map(|result| ((result.account_id, result.key), result.usage))
.collect::<BTreeMap<_, _>>();
for (account, quota) in accounts.iter().zip(quotas) {
for (item, quota) in accounts.iter().zip(quotas) {
for window in &mut quota.windows {
if window.local_usage.is_none() {
let key = (account.id.clone(), window.key.clone());
if window.local_usage.is_none()
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
{
let key = (item.account.id.clone(), window.key.clone());
if let Some(usage) = usage_by_window.get(&key) {
window.local_usage = Some(usage.clone());
}
@@ -176,33 +226,19 @@ impl DefaultAccountsService {
account_id: &ProviderAccountId,
refresh_quota: bool,
) -> Result<AccountDirectoryItem, AdminError> {
let (account, provider) = self.provider_for_account(account_id).await?;
let settings = self
.settings
.load_runtime_settings()
.await
.map_err(|error| map_store_error(error, "runtime settings"))?;
let (stored, provider) = self.provider_for_account(account_id).await?;
let account = &stored.account;
let now = Utc::now();
let retained_range = retained_usage_range(now, settings.usage_retention_days);
let rolling_range = TimeRange {
start: now - Duration::hours(24),
end: now,
};
let ids = vec![account.id.clone()];
let (usage, rolling_usage) = futures::try_join!(
async {
self.accounts
.load_account_usage(retained_range, &ids)
.await
.map_err(|error| map_store_error(error, "account usage"))
},
async {
self.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))
},
)?;
let rolling_usage = self
.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))?;
let rolling_usage = rolling_usage.into_iter().next();
let mut quota = provider
.quota(ProviderQuotaRequest {
@@ -213,17 +249,15 @@ impl DefaultAccountsService {
.await
.map_err(|error| map_provider_error(error, "provider quota"))?;
self.attach_quota_local_usage(
std::slice::from_ref(&account),
std::slice::from_ref(&stored),
std::slice::from_mut(&mut quota),
)
.await?;
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
let usage = quota.representative_window_usage().cloned();
Ok(AccountDirectoryItem {
status: signals.derive(),
error_reason,
usage: usage.into_iter().next(),
account,
projection: stored.projection,
usage,
account: stored.account,
quota,
})
}
@@ -232,13 +266,12 @@ impl DefaultAccountsService {
#[async_trait]
impl AccountsService for DefaultAccountsService {
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError> {
let (page, settings) = futures::try_join!(
self.accounts.list_accounts(query),
self.settings.load_runtime_settings(),
)
.map_err(|error| map_store_error(error, "account directory"))?;
let page = self
.accounts
.list_accounts(query)
.await
.map_err(|error| map_store_error(error, "account directory"))?;
let now = Utc::now();
let retained_range = retained_usage_range(now, settings.usage_retention_days);
let rolling_range = TimeRange {
start: now - Duration::hours(24),
end: now,
@@ -246,22 +279,19 @@ impl AccountsService for DefaultAccountsService {
let ids = page
.items
.iter()
.map(|account| account.id.clone())
.map(|item| item.account.id.clone())
.collect::<Vec<_>>();
let (usage, rolling_usage) = futures::try_join!(
self.accounts.load_account_usage(retained_range, &ids),
self.accounts.load_account_usage(rolling_range, &ids),
)
.map_err(|error| map_store_error(error, "account usage"))?;
let usage = usage
.into_iter()
.map(|usage| (usage.account_id.clone(), usage))
.collect::<BTreeMap<_, _>>();
let rolling_usage = self
.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))?;
let rolling_usage = rolling_usage
.into_iter()
.map(|usage| (usage.account_id.clone(), usage))
.collect::<BTreeMap<_, _>>();
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
let mut quotas = futures::future::join_all(page.items.iter().map(|item| async {
let account = &item.account;
let account_id = ProviderAccountId::new(account.id.clone())
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
@@ -305,14 +335,12 @@ impl AccountsService for DefaultAccountsService {
.items
.into_iter()
.zip(quotas)
.map(|(account, quota)| {
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
.map(|(item, quota)| {
let usage = quota.representative_window_usage().cloned();
AccountDirectoryItem {
status: signals.derive(),
error_reason,
usage: usage.get(&account.id).cloned(),
account,
usage,
account: item.account,
projection: item.projection,
quota,
}
})
@@ -340,7 +368,7 @@ impl AccountsService for DefaultAccountsService {
for account_id in account_ids {
let account = self.load_account(&account_id).await?;
grouped
.entry(account.provider_kind)
.entry(account.account.provider_kind)
.or_default()
.push(account_id);
}
@@ -383,7 +411,8 @@ impl AccountsService for DefaultAccountsService {
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError> {
let (account, provider) = self.provider_for_account(&account_id).await?;
let (stored, provider) = self.provider_for_account(&account_id).await?;
let account = stored.account;
let prepared = provider
.prepare_refresh(PrepareCredentialRefresh {
account: account.clone(),
@@ -409,6 +438,98 @@ impl AccountsService for DefaultAccountsService {
})
}
async fn recover(
&self,
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError> {
let (_, provider) = self.provider_for_account(&account_id).await?;
let result = self
.accounts
.recover_account(&account_id, context)
.await
.map_err(|error| map_store_error(error, "provider account recovery"))?;
provider
.account_facts_changed(std::slice::from_ref(&account_id))
.await;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
let account = self.load_directory_item(&account_id, false).await?;
Ok(AccountRefreshResult {
config_revision: result.config_revision,
account,
})
}
async fn update(
&self,
context: &MutationContext,
command: UpdateAccount,
) -> Result<AccountUpdateResult, AdminError> {
let account_id = ProviderAccountId::new(command.account_id.clone())
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
let (_, provider) = self.provider_for_account(&account_id).await?;
let enabled = command.enabled;
let result = self
.accounts
.update_account(command, context)
.await
.map_err(|error| map_store_error(error, "provider account"))?;
if !enabled {
provider.account_unavailable(&account_id).await;
}
provider
.account_facts_changed(std::slice::from_ref(&account_id))
.await;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn batch_update(
&self,
context: &MutationContext,
command: BatchUpdateAccounts,
) -> Result<AccountsUpdateResult, AdminError> {
let account_ids = command
.account_ids
.iter()
.map(|id| {
ProviderAccountId::new(id.clone())
.map_err(|_| AdminError::invalid("Invalid provider account ID"))
})
.collect::<Result<Vec<_>, _>>()?;
let mut providers = BTreeMap::<
ProviderKind,
(
Arc<dyn crate::ports::provider::ProviderAdmin>,
Vec<ProviderAccountId>,
),
>::new();
for account_id in &account_ids {
let (item, provider) = self.provider_for_account(account_id).await?;
providers
.entry(item.account.provider_kind)
.or_insert_with(|| (provider, Vec::new()))
.1
.push(account_id.clone());
}
let enabled = command.enabled;
let result = self
.accounts
.batch_update_accounts(command, context)
.await
.map_err(|error| map_store_error(error, "provider accounts"))?;
for (provider, provider_ids) in providers.values() {
if !enabled {
for account_id in provider_ids {
provider.account_unavailable(account_id).await;
}
}
provider.account_facts_changed(provider_ids).await;
}
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn quota(
&self,
account_id: &ProviderAccountId,
@@ -417,6 +538,57 @@ impl AccountsService for DefaultAccountsService {
self.load_directory_item(account_id, refresh).await
}
async fn reset_credits(
&self,
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<ProviderResetCredits, AdminError> {
let (_, provider) = self.provider_for_account(&account_id).await?;
match provider.reset_credits(&account_id).await {
Ok(credits) => Ok(credits),
Err(error)
if error.kind()
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
{
self.refresh(context, account_id.clone()).await?;
let (_, provider) = self.provider_for_account(&account_id).await?;
provider
.reset_credits(&account_id)
.await
.map_err(map_reset_credits_error_after_refresh)
}
Err(error) => Err(map_provider_error(error, "provider reset credits")),
}
}
async fn consume_reset_credit(
&self,
context: &MutationContext,
command: ConsumeProviderResetCredit,
) -> Result<ProviderResetCreditResult, AdminError> {
let account_id = command.account_id.clone();
// 覆盖 credential refresh + 同键重试的完整账号级临界区,避免 401 两次调用
// 之间插入另一笔不可逆消费。
let lock = self.reset_credit_lock(&account_id).await;
let _guard = lock.lock().await;
let (_, provider) = self.provider_for_account(&account_id).await?;
match provider.consume_reset_credit(command.clone()).await {
Ok(result) => Ok(result),
Err(error)
if error.kind()
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
{
self.refresh(context, account_id.clone()).await?;
let (_, provider) = self.provider_for_account(&account_id).await?;
provider
.consume_reset_credit(command)
.await
.map_err(map_reset_credits_error_after_refresh)
}
Err(error) => Err(map_provider_error(error, "provider reset-credit consume")),
}
}
async fn models(
&self,
account_id: &ProviderAccountId,
@@ -434,8 +606,8 @@ impl AccountsService for DefaultAccountsService {
account_id: ProviderAccountId,
upstream_model: UpstreamModelId,
) -> Result<AccountConnectionTestEventStream, AdminError> {
let (account, provider) = self.provider_for_account(&account_id).await?;
let initial_status = account_status(&account, false, Utc::now());
let (stored, provider) = self.provider_for_account(&account_id).await?;
let account = stored.account;
let model = upstream_model.as_str().to_owned();
let operation = provider
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
@@ -452,8 +624,6 @@ impl AccountsService for DefaultAccountsService {
},
];
let probe = Arc::clone(&self.probe);
let accounts = Arc::clone(&self.accounts);
let stored_account_id = account.id.clone();
let terminal = futures::stream::once(async move {
let result = probe
.probe(AccountProbeRequest {
@@ -463,22 +633,12 @@ impl AccountsService for DefaultAccountsService {
operation,
})
.await;
let status = accounts
.load_account(&stored_account_id)
.await
.ok()
.flatten()
.map_or(initial_status, |account| {
account_status(&account, false, Utc::now())
});
match result {
Ok(result) => result
.text
.into_iter()
.map(|text| AccountConnectionTestEvent::Content { text })
.chain(std::iter::once(AccountConnectionTestEvent::Completed {
account_status: status,
}))
.chain(std::iter::once(AccountConnectionTestEvent::Completed))
.collect(),
Err(error) => {
let upstream_status = error
@@ -492,11 +652,7 @@ impl AccountsService for DefaultAccountsService {
let upstream_body = error
.upstream_response()
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
let message = upstream_body
.as_deref()
.filter(|body| !body.is_empty())
.unwrap_or_else(|| error.client_message())
.to_owned();
let message = error.client_message().to_owned();
vec![AccountConnectionTestEvent::Failed {
message,
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
@@ -504,7 +660,6 @@ impl AccountsService for DefaultAccountsService {
upstream_status,
upstream_content_type,
upstream_body,
account_status: status,
}]
}
}
@@ -514,6 +669,19 @@ impl AccountsService for DefaultAccountsService {
}
}
fn map_reset_credits_error_after_refresh(
error: crate::ports::provider::ProviderAdminError,
) -> AdminError {
if error.kind() == crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired {
return AdminError::bad_gateway(
error
.message()
.unwrap_or("OpenAI reset-credit upstream rejected the refreshed credential"),
);
}
map_provider_error(error, "provider reset credits")
}
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
fn empty_quota() -> ProviderQuota {
ProviderQuota {
@@ -521,54 +689,17 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
/// 构造状态派生所需的信号快照。
fn account_signals(
account: &AccountRecord,
quota: &ProviderQuota,
now: chrono::DateTime<Utc>,
) -> AccountStatusSignals {
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: quota.limit_reached,
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
}
}
/// 账号运营分类(互斥展示状态)。
fn account_status(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatus {
let quota = ProviderQuota {
limit_reached: quota_limit_reached,
..empty_quota()
};
account_signals(account, &quota, now).derive()
}
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
TimeRange {
start: now - Duration::days(i64::from(retention_days)),
end: now,
}
}
fn quota_usage_window(
account_id: &str,
window: &ProviderQuotaWindow,
) -> Option<AccountUsageWindowQuery> {
if window.local_usage.is_some() {
if window.local_usage.is_some()
|| window.local_usage_attribution != QuotaLocalUsageAttribution::AccountWide
{
return None;
}
let reset_at = window.reset_at?;
@@ -97,7 +97,7 @@ impl ClientKeyService for DefaultClientKeyService {
id,
name: command.name,
label: command.label,
provider_kind: command.provider_kind,
group_ids: command.group_ids,
limits: command.limits,
plaintext: plaintext.clone(),
},
@@ -1,5 +1,6 @@
//! 管理控制面的用例实现。
pub mod account_groups;
pub mod accounts;
pub mod auth;
pub mod backup;
@@ -13,11 +14,11 @@ pub mod xai;
use crate::{
model::{
AdminError, AdminErrorKind, MutationContext,
accounts::{DeleteAccounts, SetAccountEnabled},
accounts::DeleteAccounts,
provider_credentials::{
AuthorizationMutationTarget, CredentialDeletion, CredentialDeletionResult,
CredentialDetails, CredentialMutation, CredentialMutationResult,
CredentialRotationCommit, PendingAuthorizationMutation, PreparedAuthorizationCommit,
CredentialDetails, CredentialMutationResult, CredentialRotationCommit,
PendingAuthorizationMutation, PreparedAuthorizationCommit,
PreparedAuthorizationCredential, PreparedCredentialImport, PreparedCredentialRotation,
StartAuthorization,
},
@@ -56,9 +57,15 @@ fn map_provider_error(
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
ProviderAdminErrorKind::CredentialRefreshRequired => AdminErrorKind::Unavailable,
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
};
AdminError::new(kind, format!("{resource} operation failed"))
let message = error
.message()
.map(ToOwned::to_owned)
.unwrap_or_else(|| format!("{resource} operation failed"));
AdminError::new(kind, message)
}
async fn publish_committed(
@@ -280,44 +287,6 @@ async fn commit_credential_refresh(
}
}
async fn set_credential_enabled(
accounts: &dyn AccountStore,
provider: &dyn ProviderAdmin,
command: CredentialMutation,
enabled: bool,
resource: &'static str,
) -> Result<CredentialMutationResult, AdminError> {
required_credential(
accounts,
provider.provider_kind(),
&command.account_id,
resource,
)
.await?;
let account_id = command.account_id;
let revision = accounts
.set_account_enabled(
SetAccountEnabled {
account_id: account_id.as_str().to_owned(),
enabled,
},
&command.context,
)
.await
.map_err(|error| map_store_error(error, resource))?;
if !enabled {
provider.account_unavailable(&account_id).await;
}
provider
.account_facts_changed(std::slice::from_ref(&account_id))
.await;
Ok(CredentialMutationResult {
config_revision: revision,
account_id,
credential_revision: None,
})
}
async fn delete_credentials(
accounts: &dyn AccountStore,
provider: &dyn ProviderAdmin,
@@ -98,10 +98,11 @@ impl ObservabilityService for DefaultObservabilityService {
range: TimeRange,
kind: TrendKind,
) -> Result<DashboardResult, AdminError> {
let observed_at = Utc::now();
let (mut observation, settings, runtime_slots) = futures::try_join!(
self.store.dashboard_summary(range),
self.store.dashboard_summary(range, observed_at),
self.settings.load_runtime_settings(),
self.store.dashboard_runtime_slots(range.end),
self.store.dashboard_runtime_slots(observed_at),
)
.map_err(|error| map_store_error(error, "dashboard"))?;
self.enrich_billing(&mut observation.recent_requests);
@@ -112,15 +113,10 @@ impl ObservabilityService for DefaultObservabilityService {
let today =
dashboard_period_metrics(&observation.trend, today_start, observation.range.end);
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
let total_billing_usd = observation
.attempts
.costs
.iter()
.find(|cost| cost.currency == "USD")
.map(|cost| cost.amount.clone());
let total_billing_usd = observation.totals.billing_usd.clone();
let total_cached_token_rate = rate_or_zero(
observation.requests.cached_tokens,
observation.requests.input_tokens,
observation.totals.cached_tokens,
observation.totals.input_tokens,
);
let average_first_token_latency_ms = average(
observation.requests.first_token_latency_sum_ms,
@@ -130,12 +126,20 @@ impl ObservabilityService for DefaultObservabilityService {
let health_timeline = health_timeline_at(&observation.trend, Utc::now());
let wire_profiles = self.providers.dashboard_wire_profiles();
let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account);
let active_accounts = runtime_slots
.as_ref()
.map_or(observation.provider_accounts.active, |slots| {
slots.active_accounts
});
let total_slots = active_accounts.saturating_mul(max_concurrent_per_account);
let total_slots = runtime_slots.as_ref().map_or_else(
|| {
observation
.provider_accounts
.normal
.saturating_mul(max_concurrent_per_account)
},
|slots| {
slots
.inherited_accounts
.saturating_mul(max_concurrent_per_account)
.saturating_add(slots.overridden_slots)
},
);
let used_slots = runtime_slots.and_then(|slots| slots.used_slots);
Ok(DashboardResult {
capacity: DashboardCapacity {
@@ -489,8 +493,7 @@ impl DefaultObservabilityService {
});
let quota_used = futures::future::join_all(quota_reads).await;
for (account, used_percent) in accounts.iter_mut().zip(quota_used) {
account.quota_used_percent = used_percent
.or_else(|| (account.availability == "quota_exhausted").then_some(100.0));
account.quota_used_percent = used_percent;
}
}
@@ -11,9 +11,9 @@ use crate::{
provider_credentials::{
AuthorizationStarted, CompleteAuthorization, CredentialDeletion,
CredentialDeletionResult, CredentialDetails, CredentialImportCommit,
CredentialImportResult, CredentialListQuery, CredentialMutation,
CredentialMutationResult, CredentialPage, ImportCredentials, PrepareCredentialImport,
PrepareCredentialRotation, ProviderQuotaRequest, RotateCredential, StartAuthorization,
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
ImportCredentials, PrepareCredentialImport, PrepareCredentialRotation,
ProviderQuotaRequest, RotateCredential, StartAuthorization,
},
},
ports::{provider::ProviderAdmin, store::AccountStore},
@@ -22,8 +22,7 @@ use crate::{
use super::{
commit_authorization, commit_credential_rotation, delete_credentials, map_provider_error,
map_store_error, pending_authorization, publish_committed, required_credential,
set_credential_enabled, validate_authorization_commit, validate_prepared_import,
validate_prepared_rotation,
validate_authorization_commit, validate_prepared_import, validate_prepared_rotation,
};
/// OpenAI 固定管理路由消费的服务。
@@ -50,14 +49,6 @@ pub trait OpenAiService: Send + Sync {
&self,
command: RotateCredential,
) -> Result<CredentialMutationResult, AdminError>;
async fn enable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError>;
async fn disable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError>;
async fn delete(
&self,
command: CredentialDeletion,
@@ -84,25 +75,29 @@ impl DefaultOpenAiService {
}
}
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
for account_id in account_ids {
if let Err(error) = self
.provider
.quota(ProviderQuotaRequest {
account_id: account_id.clone(),
refresh: true,
rolling_usage: None,
})
.await
{
tracing::warn!(
request_id,
account_id = %account_id.as_str(),
quota_error = ?error.kind(),
"OpenAI initial quota observation failed"
);
fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
let provider = Arc::clone(&self.provider);
let account_ids = account_ids.to_vec();
let request_id = request_id.to_owned();
tokio::spawn(async move {
for account_id in account_ids {
if let Err(error) = provider
.quota(ProviderQuotaRequest {
account_id: account_id.clone(),
refresh: true,
rolling_usage: None,
})
.await
{
tracing::warn!(
request_id,
account_id = %account_id.as_str(),
quota_error = ?error.kind(),
"OpenAI initial quota observation failed"
);
}
}
}
});
}
}
@@ -155,8 +150,7 @@ impl OpenAiService for DefaultOpenAiService {
.await;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
.await;
self.observe_initial_quotas(&result.credential_ids, &context.request_id);
Ok(result)
}
@@ -208,8 +202,7 @@ impl OpenAiService for DefaultOpenAiService {
self.observe_initial_quotas(
std::slice::from_ref(&result.account_id),
&context.request_id,
)
.await;
);
Ok(result)
}
@@ -250,38 +243,6 @@ impl OpenAiService for DefaultOpenAiService {
Ok(result)
}
async fn enable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError> {
let result = set_credential_enabled(
self.accounts.as_ref(),
self.provider.as_ref(),
command,
true,
"OpenAI credential",
)
.await?;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn disable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError> {
let result = set_credential_enabled(
self.accounts.as_ref(),
self.provider.as_ref(),
command,
false,
"OpenAI credential",
)
.await?;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn delete(
&self,
command: CredentialDeletion,
@@ -11,8 +11,8 @@ use crate::{
provider_credentials::{
AuthorizationStarted, CompleteAuthorization, CredentialDeletion,
CredentialDeletionResult, CredentialImportCommit, CredentialImportResult,
CredentialListQuery, CredentialMutation, CredentialMutationResult, CredentialPage,
ImportCredentials, PrepareCredentialImport, ProviderQuotaRequest, StartAuthorization,
CredentialListQuery, CredentialMutationResult, CredentialPage, ImportCredentials,
PrepareCredentialImport, ProviderQuotaRequest, StartAuthorization,
},
},
ports::{provider::ProviderAdmin, store::AccountStore},
@@ -20,8 +20,8 @@ use crate::{
use super::{
commit_authorization, delete_credentials, map_provider_error, map_store_error,
pending_authorization, publish_committed, set_credential_enabled,
validate_authorization_commit, validate_prepared_import,
pending_authorization, publish_committed, validate_authorization_commit,
validate_prepared_import,
};
/// xAI 固定管理路由消费的服务。
@@ -40,14 +40,6 @@ pub trait XaiService: Send + Sync {
&self,
command: CompleteAuthorization,
) -> Result<CredentialMutationResult, AdminError>;
async fn enable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError>;
async fn disable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError>;
async fn delete(
&self,
command: CredentialDeletion,
@@ -166,38 +158,6 @@ impl XaiService for DefaultXaiService {
Ok(result)
}
async fn enable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError> {
let result = set_credential_enabled(
self.accounts.as_ref(),
self.provider.as_ref(),
command,
true,
"xAI credential",
)
.await?;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn disable(
&self,
command: CredentialMutation,
) -> Result<CredentialMutationResult, AdminError> {
let result = set_credential_enabled(
self.accounts.as_ref(),
self.provider.as_ref(),
command,
false,
"xAI credential",
)
.await?;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
Ok(result)
}
async fn delete(
&self,
command: CredentialDeletion,
@@ -13,9 +13,10 @@ use gateway_admin::{
MutationActor, MutationContext, Revision,
auth::AdminAuditEvent,
backup::{
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
UpdateBackupStorageCommand,
},
},
ports::{
@@ -497,15 +498,23 @@ impl DatabaseDumpPort for FakeDumpPort {
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
pub(crate) struct FakeObjectStore {
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
upload_error: Option<BackupError>,
}
impl FakeObjectStore {
pub(crate) fn new() -> Self {
Self {
objects: Mutex::new(std::collections::HashMap::new()),
upload_error: None,
}
}
/// 让上传返回指定的脱敏 S3 错误。
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
self.upload_error = Some(BackupError::new(code, message.into()));
self
}
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
self.objects.lock().expect("objects").get(key).cloned()
}
@@ -530,6 +539,9 @@ impl BackupObjectStorePort for FakeObjectStore {
_config: &BackupStorageConfig,
request: UploadObjectRequest,
) -> Result<(), gateway_admin::model::backup::BackupError> {
if let Some(error) = &self.upload_error {
return Err(error.clone());
}
let content = std::fs::read(&request.source).map_err(|_| {
gateway_admin::model::backup::BackupError::new(
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
@@ -6,9 +6,9 @@ use chrono::{Duration, Utc};
use gateway_admin::{
backup::task::BackupTask,
model::backup::{
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
},
ports::backup::BackupRepository,
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
};
use gateway_core::engine::CancellationToken;
@@ -89,6 +89,78 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
assert!(object_store.object(&seed.object_key).is_none());
}
#[tokio::test]
async fn backup_task_marks_failed_with_the_upload_error() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store =
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
let seed = BackupRecordSeed {
id: backup_id("upload-failure"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let task = BackupTask::new(repository.clone(), dump, object_store);
task.run_cycle(&CancellationToken::new())
.await
.expect("run cycle");
let records = repository.all_records();
let failed = &records[0];
assert_eq!(failed.status, BackupStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
assert_eq!(
failed.error_message.as_deref(),
Some("上传到对象存储失败:对象存储拒绝上传")
);
}
#[tokio::test]
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store =
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
let seed = BackupRecordSeed {
id: backup_id("recovered-upload-failure"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let cancellation = CancellationToken::new();
dump.dump(DumpRequest {
backup_id: seed.id.clone(),
cancellation: cancellation.clone(),
})
.await
.expect("stage dump");
repository.force_status(&seed.id, BackupStatus::Dumping);
let task = BackupTask::new(repository.clone(), dump, object_store);
task.run_cycle(&cancellation).await.expect("run cycle");
let records = repository.all_records();
let failed = &records[0];
assert_eq!(failed.status, BackupStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
assert_eq!(
failed.error_message.as_deref(),
Some("上传到对象存储失败:对象存储拒绝上传")
);
}
#[tokio::test]
async fn retention_cleans_expired_scheduled_backups() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
+115 -2
View File
@@ -6,11 +6,13 @@ use gateway_core::engine::credential::OpaqueProviderData;
use gateway_admin::{
model::{
PageSize, Revision,
accounts::AccountUsage,
auth::LoginCommand,
client_keys::ClientKeyPageSize,
observability::{DecimalAmount, RequestOutcome, TimeRange},
provider_credentials::{
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
QuotaLocalUsageAttribution,
},
settings::AdminApiKey,
},
@@ -128,13 +130,121 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
quota_window("shortTerm", Some(14_400), Some(45.0)),
],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
};
assert_eq!(quota.representative_used_percent(), Some(45.0));
}
#[test]
fn representative_quota_should_prefer_account_wide_window_over_model_specific_window() {
let mut model_specific = quota_window("shortTerm", Some(18_000), Some(64.0));
model_specific.local_usage_attribution = QuotaLocalUsageAttribution::Unavailable;
let quota = ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: vec![
model_specific,
quota_window("shortTerm", Some(604_800), Some(5.0)),
],
limit_reached: false,
provider_data: None,
};
assert_eq!(quota.representative_used_percent(), Some(5.0));
}
#[test]
fn representative_window_usage_should_accept_account_wide_rolling_window_without_reset() {
let usage = AccountUsage {
account_id: "acct_xai".to_owned(),
request_count: 5,
success_count: 5,
input_tokens: Some(100),
output_tokens: Some(20),
cached_tokens: Some(80),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
image_input_tokens: Some(0),
image_output_tokens: Some(0),
image_request_count: 0,
image_request_failed_count: 0,
total_tokens: Some(120),
cost_coverage: Default::default(),
costs: Vec::new(),
last_used_at: Some(Utc::now()),
request_buckets: Vec::new(),
models: Vec::new(),
};
let quota = ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: vec![ProviderQuotaWindow {
key: "free-rolling-24h".to_owned(),
group: "shortTerm".to_owned(),
label: "日限额".to_owned(),
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(86_400),
used_percent: None,
reset_at: None,
limit_reached: false,
local_usage: Some(usage.clone()),
provider_data: None,
}],
limit_reached: false,
provider_data: None,
};
assert_eq!(quota.representative_window_usage(), Some(&usage));
}
#[test]
fn exhausted_quota_should_project_full_usage_to_only_the_representative_window() {
let mut quota = ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: vec![
quota_window("monthly", Some(2_592_000), Some(99.0)),
quota_window("shortTerm", Some(604_800), Some(80.0)),
quota_window("shortTerm", Some(18_000), Some(30.0)),
quota_window("shortTerm", Some(14_400), Some(95.0)),
],
limit_reached: true,
provider_data: None,
};
assert_eq!(quota.representative_used_percent(), Some(100.0));
quota.apply_limit_reached_display();
assert_eq!(quota.windows[0].used_percent, Some(99.0));
assert_eq!(quota.windows[1].used_percent, Some(80.0));
assert_eq!(quota.windows[2].used_percent, Some(30.0));
assert_eq!(quota.windows[3].used_percent, Some(100.0));
assert!(quota.windows[3].limit_reached);
}
#[test]
fn exhausted_quota_should_preserve_the_provider_identified_reached_window() {
let mut reached = quota_window("monthly", Some(2_592_000), Some(98.0));
reached.limit_reached = true;
let mut quota = ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: vec![reached, quota_window("shortTerm", Some(18_000), Some(95.0))],
limit_reached: true,
provider_data: None,
};
quota.apply_limit_reached_display();
assert_eq!(quota.windows[0].used_percent, Some(100.0));
assert_eq!(quota.windows[1].used_percent, Some(95.0));
assert!(!quota.windows[1].limit_reached);
}
fn quota_window(
group: &str,
window_seconds: Option<u64>,
@@ -144,7 +254,10 @@ fn quota_window(
key: group.to_owned(),
group: group.to_owned(),
label: group.to_owned(),
source: None,
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds,
used_percent,
reset_at: None,
@@ -5,7 +5,10 @@ use chrono::{TimeDelta, Utc};
use futures::{StreamExt as _, future::BoxFuture};
use gateway_core::{
engine::{
credential::{OpaqueProviderData, ProviderAccountId},
credential::{
AccountStatusFacts, CredentialState, OpaqueProviderData, ProviderAccountId,
QuotaEvidence, QuotaState, resolve_account_status,
},
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
},
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
@@ -18,22 +21,25 @@ use gateway_admin::{
model::{
AdminError, MutationContext, Revision,
accounts::{
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
AccountConnectionTestEvent, AccountListQuery, AccountPage, AccountPageItem,
AccountRecord, AccountSummary, AccountUpdateResult, AccountUsage,
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
},
observability::TimeRange,
provider_credentials::{
AuthorizationCommit, AuthorizationCommitGuard, AuthorizationCredentialCommit,
AuthorizationMutationTarget, AuthorizationStarted, CompleteAuthorization,
CredentialCommitGuard, CredentialDetails, CredentialImportCommit,
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
CredentialRotationCommit, PendingAuthorizationMutation, PrepareCredentialImport,
PrepareCredentialRefresh, PrepareCredentialRotation, PreparedAuthorizationCommit,
ConsumeProviderResetCredit, CredentialCommitGuard, CredentialDetails,
CredentialImportCommit, CredentialImportResult, CredentialListQuery,
CredentialMutationResult, CredentialPage, CredentialRotationCommit,
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
PrepareCredentialRotation, PreparedAuthorizationCommit,
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
ProviderQuotaRequest, ProviderQuotaWindow,
ProviderQuotaRequest, ProviderQuotaWindow, ProviderResetCreditResult,
QuotaLocalUsageAttribution,
},
settings::{
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
@@ -56,7 +62,7 @@ pub(super) type EventLog = Arc<Mutex<Vec<&'static str>>>;
pub(super) struct FakeProviderAdmin {
kind: ProviderKind,
events: EventLog,
failure: Mutex<Option<ProviderAdminErrorKind>>,
failure: Mutex<Option<ProviderAdminError>>,
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
retry_authorization_after_abort: Mutex<bool>,
@@ -65,6 +71,7 @@ pub(super) struct FakeProviderAdmin {
quota_requests: Mutex<Vec<ProviderQuotaRequest>>,
quota: Mutex<ProviderQuota>,
current_credential_revision: Mutex<Revision>,
reset_credit_commands: Mutex<Vec<ConsumeProviderResetCredit>>,
}
impl FakeProviderAdmin {
@@ -81,11 +88,17 @@ impl FakeProviderAdmin {
quota_requests: Mutex::new(Vec::new()),
quota: Mutex::new(empty_quota()),
current_credential_revision: Mutex::new(revision(1)),
reset_credit_commands: Mutex::new(Vec::new()),
})
}
pub(super) fn fail_next(&self, kind: ProviderAdminErrorKind) {
*self.failure.lock().expect("provider failure") = Some(kind);
*self.failure.lock().expect("provider failure") = Some(ProviderAdminError::new(kind));
}
fn fail_next_with_message(&self, kind: ProviderAdminErrorKind, message: &str) {
*self.failure.lock().expect("provider failure") =
Some(ProviderAdminError::new(kind).with_message(message));
}
pub(super) fn fail_next_quota(&self, kind: ProviderAdminErrorKind) {
@@ -109,6 +122,13 @@ impl FakeProviderAdmin {
.clone()
}
fn reset_credit_commands(&self) -> Vec<ConsumeProviderResetCredit> {
self.reset_credit_commands
.lock()
.expect("provider reset-credit commands")
.clone()
}
pub(super) fn set_quota(&self, quota: ProviderQuota) {
*self.quota.lock().expect("provider quota") = quota;
}
@@ -141,7 +161,7 @@ impl FakeProviderAdmin {
fn require_available(&self) -> Result<(), ProviderAdminError> {
match self.failure.lock().expect("provider failure").take() {
Some(kind) => Err(ProviderAdminError::new(kind)),
Some(error) => Err(error),
None => Ok(()),
}
}
@@ -340,6 +360,22 @@ impl ProviderAdmin for FakeProviderAdmin {
Ok(self.quota.lock().expect("provider quota").clone())
}
async fn consume_reset_credit(
&self,
command: ConsumeProviderResetCredit,
) -> Result<ProviderResetCreditResult, ProviderAdminError> {
self.record("provider.consume_reset_credit");
self.reset_credit_commands
.lock()
.expect("provider reset-credit commands")
.push(command);
self.require_available()?;
Ok(ProviderResetCreditResult {
code: "reset".to_owned(),
credit: None,
})
}
async fn models(
&self,
_: &ProviderAccountId,
@@ -378,6 +414,7 @@ pub(super) struct FakeAccountStore {
fail_commit: Mutex<bool>,
audit_requests: Mutex<Vec<String>>,
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
}
impl FakeAccountStore {
@@ -393,6 +430,7 @@ impl FakeAccountStore {
fail_commit: Mutex::new(false),
audit_requests: Mutex::new(Vec::new()),
quota_window_usage: Mutex::new(Vec::new()),
quota_window_queries: Mutex::new(Vec::new()),
})
}
@@ -408,6 +446,13 @@ impl FakeAccountStore {
*self.quota_window_usage.lock().expect("quota window usage") = usage;
}
fn quota_window_queries(&self) -> Vec<AccountUsageWindowQuery> {
self.quota_window_queries
.lock()
.expect("quota window queries")
.clone()
}
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
*self.accounts.lock().expect("accounts") = accounts;
}
@@ -438,6 +483,22 @@ impl FakeAccountStore {
.expect("audit requests")
.push(context.request_id.clone());
}
fn page_item(account: AccountRecord) -> AccountPageItem {
let facts = AccountStatusFacts {
enabled: account.enabled,
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until: None,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message.clone(),
};
AccountPageItem {
account,
projection: resolve_account_status(&facts, std::time::SystemTime::now()),
}
}
}
#[async_trait]
@@ -445,12 +506,13 @@ impl AccountStore for FakeAccountStore {
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
self.record("store.list_accounts");
let accounts = self.accounts.lock().expect("accounts").clone();
let total = accounts.len() as u64;
Ok(AccountPage {
config_revision: revision(1),
items: accounts.clone(),
total: accounts.len() as u64,
items: accounts.into_iter().map(Self::page_item).collect(),
total,
summary: AccountSummary {
total: accounts.len() as u64,
total,
normal: 1,
quota_exhausted: 0,
rate_limited: 0,
@@ -460,7 +522,7 @@ impl AccountStore for FakeAccountStore {
})
}
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
self.record("store.load_account");
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
let account = self
@@ -477,7 +539,7 @@ impl AccountStore for FakeAccountStore {
.find(|account| account.id == account_id)
.cloned()
});
Ok(account)
Ok(account.map(Self::page_item))
}
async fn load_account_usage(
@@ -490,8 +552,12 @@ impl AccountStore for FakeAccountStore {
async fn load_account_usage_by_windows(
&self,
_: &[AccountUsageWindowQuery],
windows: &[AccountUsageWindowQuery],
) -> AdminStoreResult<Vec<AccountUsageWindowResult>> {
*self
.quota_window_queries
.lock()
.expect("quota window queries") = windows.to_vec();
Ok(self
.quota_window_usage
.lock()
@@ -628,15 +694,61 @@ impl AccountStore for FakeAccountStore {
Ok(rotation_result(command))
}
async fn set_account_enabled(
async fn update_account(
&self,
_: SetAccountEnabled,
command: UpdateAccount,
context: &MutationContext,
) -> AdminStoreResult<Revision> {
self.record("store.set_enabled");
) -> AdminStoreResult<AccountUpdateResult> {
self.record("store.update_account");
self.record_context(context);
self.require_commit()?;
Ok(revision(2))
Ok(AccountUpdateResult {
config_revision: revision(2),
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
})
}
async fn recover_account(
&self,
account_id: &ProviderAccountId,
context: &MutationContext,
) -> AdminStoreResult<AccountUpdateResult> {
self.record("store.recover_account");
self.record_context(context);
self.require_commit()?;
let mut accounts = self.accounts.lock().expect("accounts");
if let Some(account) = accounts
.iter_mut()
.find(|account| account.id == account_id.as_str())
{
account.enabled = true;
account.credential_state = CredentialState::Ready;
account.quota = QuotaState::allowed(std::time::SystemTime::now());
account.last_error_reason = None;
account.last_error_message = None;
}
Ok(AccountUpdateResult {
config_revision: revision(2),
account_id: account_id.clone(),
})
}
async fn batch_update_accounts(
&self,
command: BatchUpdateAccounts,
context: &MutationContext,
) -> AdminStoreResult<AccountsUpdateResult> {
self.record("store.batch_update_accounts");
self.record_context(context);
self.require_commit()?;
Ok(AccountsUpdateResult {
config_revision: revision(2),
account_ids: command
.account_ids
.into_iter()
.map(|id| ProviderAccountId::new(id).expect("account ID"))
.collect(),
})
}
async fn delete_accounts(
@@ -815,7 +927,11 @@ fn provider_registry_should_reject_duplicate_kind() {
async fn connection_test_should_probe_unavailable_account() {
let provider = FakeProviderAdmin::new("xai", events());
let mut account = account_record("xai");
account.availability = AccountAvailability::QuotaExhausted;
account.quota = QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
std::time::SystemTime::now(),
None,
);
let store = FakeAccountStore::with_account(account, events());
let services =
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
@@ -837,7 +953,6 @@ async fn connection_test_should_probe_unavailable_account() {
message,
provider_error_code: Some(code),
provider_error_type: Some(error_type),
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
..
}) if message == "included usage exhausted"
&& code == "usage_exhausted"
@@ -846,7 +961,7 @@ async fn connection_test_should_probe_unavailable_account() {
}
#[tokio::test]
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
async fn connection_test_rate_limited_probe_returns_provider_failure() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
@@ -869,9 +984,10 @@ async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
message,
..
})
if message == "xAI free model quota is exhausted"
));
}
@@ -897,10 +1013,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
..
})
Some(AccountConnectionTestEvent::Failed { .. })
));
}
@@ -949,7 +1062,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
assert_eq!(result.config_revision, revision(2));
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
assert_eq!(
result.account.status,
result.account.projection.status,
gateway_admin::model::accounts::AccountStatus::Normal
);
@@ -967,6 +1080,175 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
assert_eq!(store.audit_requests(), ["refresh-request"]);
}
#[tokio::test]
async fn accounts_recover_should_commit_facts_then_return_normal_account() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let mut account = account_record("openai");
account.enabled = false;
account.credential_state = CredentialState::Invalid;
account.quota = QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
std::time::SystemTime::now(),
None,
);
account.last_error_reason =
Some(gateway_core::engine::credential::AccountErrorReason::CredentialInvalid);
account.last_error_message = Some("invalid credential".to_owned());
let store = FakeAccountStore::with_account(account, events.clone());
let services = accounts_service(provider, store.clone()).await;
let result = services
.accounts()
.recover(
&context("recover-request"),
ProviderAccountId::new("acct_test").expect("account ID"),
)
.await
.expect("recover account");
assert_eq!(result.config_revision, revision(2));
assert!(result.account.account.enabled);
assert_eq!(
result.account.account.credential_state,
CredentialState::Ready
);
assert_eq!(
result.account.account.quota.access(),
gateway_core::engine::credential::QuotaAccessState::Allowed
);
assert_eq!(
result.account.projection.status,
gateway_admin::model::accounts::AccountStatus::Normal
);
assert_eq!(
recorded(&events),
[
"store.load_account",
"store.recover_account",
"provider.account_facts_changed",
"store.load_account",
]
);
assert_eq!(store.audit_requests(), ["recover-request"]);
}
#[tokio::test]
async fn accounts_update_should_commit_then_release_disabled_account_and_publish_facts() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
let services = accounts_service(provider, store.clone()).await;
let result = services
.accounts()
.update(
&context("update-request"),
UpdateAccount {
account_id: "acct_test".to_owned(),
enabled: false,
concurrency_limit: None,
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
group_ids: Vec::new(),
},
)
.await
.expect("update account");
assert_eq!(result.config_revision, revision(2));
assert_eq!(
recorded(&events),
[
"store.load_account",
"store.update_account",
"provider.account_unavailable",
"provider.account_facts_changed",
]
);
assert_eq!(store.audit_requests(), ["update-request"]);
}
#[tokio::test]
async fn accounts_update_should_not_notify_provider_when_store_commit_fails() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
store.fail_next_commit();
let services = accounts_service(provider, store).await;
services
.accounts()
.update(
&context("update-failure"),
UpdateAccount {
account_id: "acct_test".to_owned(),
enabled: false,
concurrency_limit: None,
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
group_ids: Vec::new(),
},
)
.await
.expect_err("failed account update");
assert_eq!(
recorded(&events),
["store.load_account", "store.update_account"]
);
}
#[tokio::test]
async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
let events = events();
let openai = FakeProviderAdmin::new("openai", events.clone());
let xai = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
let mut openai_account = account_record("openai");
openai_account.id = "acct_openai".to_owned();
let mut xai_account = account_record("xai");
xai_account.id = "acct_xai".to_owned();
store.set_accounts(vec![openai_account, xai_account]);
let services = super::AdminHarness::new()
.accounts(store.clone())
.settings(Arc::new(StaticSettingsStore))
.provider(openai)
.provider(xai)
.probe(Arc::new(SuccessfulAccountProbe))
.build()
.await;
let result = services
.accounts()
.batch_update(
&context("batch-update-request"),
BatchUpdateAccounts {
account_ids: vec!["acct_openai".to_owned(), "acct_xai".to_owned()],
enabled: false,
concurrency_limit: None,
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
group_ids: Vec::new(),
},
)
.await
.expect("batch update accounts");
assert_eq!(result.config_revision, revision(2));
assert_eq!(result.account_ids.len(), 2);
assert_eq!(store.audit_requests(), ["batch-update-request"]);
assert_eq!(
recorded(&events),
[
"store.load_account",
"store.load_account",
"store.batch_update_accounts",
"provider.account_unavailable",
"provider.account_facts_changed",
"provider.account_unavailable",
"provider.account_facts_changed",
]
);
}
#[tokio::test]
async fn accounts_list_should_return_complete_directory_semantics() {
let provider = FakeProviderAdmin::new("openai", events());
@@ -978,6 +1260,7 @@ async fn accounts_list_should_return_complete_directory_semantics() {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
@@ -990,7 +1273,7 @@ async fn accounts_list_should_return_complete_directory_semantics() {
let account = page.items.first().expect("account item");
assert_eq!(account.account.provider_kind.as_str(), "openai");
assert_eq!(
account.status,
account.projection.status,
gateway_admin::model::accounts::AccountStatus::Normal
);
}
@@ -1006,7 +1289,10 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(97.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
@@ -1015,7 +1301,6 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let failing = FakeProviderAdmin::new("xai", events.clone());
@@ -1039,6 +1324,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
@@ -1063,10 +1349,14 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
}
#[tokio::test]
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
async fn accounts_list_should_prefer_credential_error_over_quota_exhaustion() {
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::QuotaExhausted;
account.quota = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
std::time::SystemTime::now(),
None,
);
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
let store = FakeAccountStore::with_account(account, events());
@@ -1077,6 +1367,7 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
@@ -1085,17 +1376,17 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
.expect("usage-limited account directory");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
#[tokio::test]
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
async fn accounts_list_should_map_unknown_credential_to_error_not_normal() {
// Unknown 不可调度,Admin 不得显示为 normal。
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Unknown;
account.credential_state = CredentialState::Unknown;
let store = FakeAccountStore::with_account(account, events());
let page = accounts_service(provider, store)
@@ -1105,6 +1396,7 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
@@ -1113,15 +1405,13 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
.expect("unknown account directory");
assert_eq!(
page.items.first().expect("account item").status,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
#[tokio::test]
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
async fn accounts_list_should_not_derive_rate_limited_from_provider_quota_view() {
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
@@ -1130,7 +1420,10 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(100.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
@@ -1139,7 +1432,6 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
provider_data: None,
}],
limit_reached: true,
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1151,22 +1443,22 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
})
.await
.expect("rate limited account directory");
.expect("account directory with display quota cooldown");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::RateLimited,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Normal,
);
}
#[tokio::test]
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
async fn accounts_list_should_not_derive_exhaustion_from_provider_quota_view() {
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
@@ -1175,7 +1467,10 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(100.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
@@ -1184,7 +1479,6 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
provider_data: None,
}],
limit_reached: true,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1196,16 +1490,17 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
})
.await
.expect("quota exhausted account directory");
.expect("account directory with display quota limit");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Normal,
);
}
@@ -1220,7 +1515,10 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(97.0),
reset_at: Some(reset_at),
@@ -1229,7 +1527,6 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1239,13 +1536,14 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
usage: quota_local_usage("acct_test", 4_330_000),
}]);
let page = accounts_service(provider, store)
let page = accounts_service(provider, store.clone())
.await
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
@@ -1253,11 +1551,124 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
.await
.expect("quota window usage");
let usage = page.items[0].quota.windows[0]
let item = &page.items[0];
let usage = item.quota.windows[0]
.local_usage
.as_ref()
.expect("quota window local usage");
assert_eq!(usage.total_tokens, Some(4_330_000));
assert_eq!(
item.usage.as_ref().and_then(|usage| usage.total_tokens),
Some(4_330_000),
);
let queries = store.quota_window_queries();
assert_eq!(queries.len(), 1);
assert_eq!(queries[0].account_id, "acct_test");
assert_eq!(queries[0].key, "primary");
assert_eq!(queries[0].range.start, reset_at - TimeDelta::hours(5));
assert_eq!(queries[0].range.end, reset_at);
}
#[tokio::test]
async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_windows() {
let provider = FakeProviderAdmin::new("openai", events());
let reset_at = Utc::now() + TimeDelta::days(7);
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
refresh_token_expires_at: None,
windows: vec![
ProviderQuotaWindow {
key: "core-primary".to_owned(),
group: "weekly".to_owned(),
label: "周额度".to_owned(),
limit_id: Some("codex".to_owned()),
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(7 * 24 * 60 * 60),
used_percent: Some(1.0),
reset_at: Some(reset_at),
limit_reached: false,
local_usage: None,
provider_data: None,
},
ProviderQuotaWindow {
key: "codex-bengalfox-primary".to_owned(),
group: "weekly".to_owned(),
label: "周额度".to_owned(),
limit_id: Some("codex_bengalfox".to_owned()),
limit_name: Some("GPT-5.3-Codex-Spark".to_owned()),
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
window_seconds: Some(7 * 24 * 60 * 60),
used_percent: Some(0.0),
reset_at: Some(reset_at),
limit_reached: false,
local_usage: None,
provider_data: None,
},
],
limit_reached: false,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
store.set_quota_window_usage(vec![
AccountUsageWindowResult {
account_id: "acct_test".to_owned(),
key: "core-primary".to_owned(),
usage: quota_local_usage("acct_test", 12_818_806),
},
AccountUsageWindowResult {
account_id: "acct_test".to_owned(),
key: "codex-bengalfox-primary".to_owned(),
usage: quota_local_usage("acct_test", 127_926),
},
]);
let page = accounts_service(provider, store)
.await
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: None,
})
.await
.expect("quota window usage");
let local_tokens = page.items[0]
.quota
.windows
.iter()
.map(|window| {
(
window.key.as_str(),
window
.local_usage
.as_ref()
.and_then(|usage| usage.total_tokens),
)
})
.collect::<Vec<_>>();
assert_eq!(
local_tokens,
[
("core-primary", Some(12_818_806)),
("codex-bengalfox-primary", None),
],
);
assert_eq!(
page.items[0]
.usage
.as_ref()
.and_then(|usage| usage.total_tokens),
Some(12_818_806),
);
}
#[tokio::test]
@@ -1283,6 +1694,59 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
);
}
#[tokio::test]
async fn reset_credit_oauth_refresh_should_reuse_the_exact_consume_command() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
provider.fail_next(ProviderAdminErrorKind::CredentialRefreshRequired);
let store = FakeAccountStore::new("openai", events.clone());
let services = accounts_service(provider.clone(), store).await;
let redeem_request_id =
uuid::Uuid::parse_str("8fbf302d-11df-4bd5-82e4-08e4b3df7874").expect("UUID v4");
let command = ConsumeProviderResetCredit {
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
credit_id: Some("credit_1".to_owned()),
redeem_request_id,
};
let result = services
.accounts()
.consume_reset_credit(&context("reset-credit-refresh"), command.clone())
.await
.expect("consume after OAuth refresh");
assert_eq!(result.code, "reset");
assert_eq!(provider.reset_credit_commands(), [command.clone(), command]);
assert!(recorded(&events).contains(&"provider.prepare_refresh"));
assert!(recorded(&events).contains(&"store.commit_refresh"));
}
#[tokio::test]
async fn accounts_refresh_should_preserve_the_provider_failure_message() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let upstream_message = "Your refresh token has already been used.";
provider.fail_next_with_message(ProviderAdminErrorKind::Conflict, upstream_message);
let store = FakeAccountStore::new("openai", events.clone());
let services = accounts_service(provider, store).await;
let error = services
.accounts()
.refresh(
&context("refresh-provider-message"),
ProviderAccountId::new("acct_test").expect("account ID"),
)
.await
.expect_err("Provider preparation must fail");
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
assert_eq!(error.to_string(), upstream_message);
assert_eq!(
recorded(&events),
["store.load_account", "provider.prepare_refresh"]
);
}
#[tokio::test]
async fn accounts_refresh_store_failure_should_drop_guard_after_commit_attempt() {
let events = events();
@@ -1338,7 +1802,6 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
@@ -1371,9 +1834,10 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
AccountRecord {
id: "acct_test".to_owned(),
provider_kind: ProviderKind::new(kind).expect("provider kind"),
groups: Vec::new(),
name: "test account".to_owned(),
email: Some("test@example.invalid".to_owned()),
upstream_user_id: "upstream-user".to_owned(),
upstream_user_id: Some("upstream-user".to_owned()),
upstream_account_id: None,
plan_type: Some("test".to_owned()),
authentication_kind: "oauth".to_owned(),
@@ -1382,10 +1846,13 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
access_token_expires_at: Some(now + TimeDelta::hours(1)),
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
concurrency_limit: None,
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
credential_state: CredentialState::Ready,
credential_observed_at: now,
quota: QuotaState::allowed(now.into()),
last_error_reason: None,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
}
@@ -1410,7 +1877,7 @@ fn prepared_create_with_id(
provider_kind,
name: name.to_owned(),
email: Some("prepared@example.invalid".to_owned()),
upstream_user_id: "prepared-user".to_owned(),
upstream_user_id: Some("prepared-user".to_owned()),
upstream_account_id: None,
plan_type: Some("test".to_owned()),
authentication_kind: "oauth".to_owned(),
@@ -1419,8 +1886,8 @@ fn prepared_create_with_id(
access_token_expires_at: Some(now + TimeDelta::hours(1)),
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
credential_state: CredentialState::Ready,
credential_observed_at: now,
}
}
@@ -1503,9 +1970,7 @@ impl AccountProbe for FreeModelQuotaProbe {
_: AccountProbeRequest,
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async move {
let mut account = account_record("xai");
account.availability = AccountAvailability::Ready;
self.store.set_account_after_probe(account);
self.store.set_account_after_probe(account_record("xai"));
Err(GatewayError::new(
GatewayErrorKind::RateLimited,
"xAI free model quota is exhausted",
@@ -14,14 +14,20 @@ use std::{
};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures::future::BoxFuture;
use gateway_admin::{
AdminConfig, AdminServices, InitialAdminPassword,
model::{
MutationContext, Revision,
account_groups::{
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, DeleteAccountGroup,
NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
},
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
AccountListQuery, AccountPage, AccountUpdateResult, AccountUsage,
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
},
auth::{AdminAuditEvent, AdminSession},
client_keys::{
@@ -50,8 +56,8 @@ use gateway_admin::{
backup::BackupStorePorts,
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
store::{
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
AccountGroupStore, AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts,
AdminStoreResult, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
},
system::{
SystemOperationError, SystemOperationErrorKind, SystemOperations,
@@ -170,6 +176,7 @@ impl AdminHarness {
},
AdminStorePorts::new(
self.accounts,
Arc::new(UnavailableAccountGroupStore),
self.auth,
self.client_keys,
self.observability,
@@ -234,13 +241,60 @@ impl AuthStore for BootstrapAuthStore {
struct UnavailableStore;
struct UnavailableAccountGroupStore;
#[async_trait]
impl AccountGroupStore for UnavailableAccountGroupStore {
async fn list_account_groups(
&self,
_: AccountGroupListQuery,
) -> AdminStoreResult<AccountGroupPage> {
Err(unavailable("account groups"))
}
async fn create_account_group(
&self,
_: NewAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
Err(unavailable("account group create"))
}
async fn update_account_group(
&self,
_: UpdateAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
Err(unavailable("account group update"))
}
async fn set_account_group_enabled(
&self,
_: SetAccountGroupEnabled,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
Err(unavailable("account group state"))
}
async fn delete_account_group(
&self,
_: DeleteAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
Err(unavailable("account group delete"))
}
}
#[async_trait]
impl AccountStore for UnavailableStore {
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
Err(unavailable("accounts"))
}
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(
&self,
_: &str,
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountPageItem>> {
Err(unavailable("account"))
}
@@ -315,14 +369,30 @@ impl AccountStore for UnavailableStore {
Err(unavailable("credential refresh"))
}
async fn set_account_enabled(
async fn update_account(
&self,
_: SetAccountEnabled,
_: UpdateAccount,
_: &MutationContext,
) -> AdminStoreResult<Revision> {
) -> AdminStoreResult<AccountUpdateResult> {
Err(unavailable("account enabled"))
}
async fn recover_account(
&self,
_: &ProviderAccountId,
_: &MutationContext,
) -> AdminStoreResult<AccountUpdateResult> {
Err(unavailable("account recovery"))
}
async fn batch_update_accounts(
&self,
_: BatchUpdateAccounts,
_: &MutationContext,
) -> AdminStoreResult<AccountsUpdateResult> {
Err(unavailable("account batch update"))
}
async fn delete_accounts(
&self,
_: DeleteAccounts,
@@ -388,7 +458,11 @@ impl ClientKeyStore for UnavailableStore {
#[async_trait]
impl ObservabilityStore for UnavailableStore {
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
async fn dashboard_summary(
&self,
_: TimeRange,
_: DateTime<Utc>,
) -> AdminStoreResult<DashboardObservation> {
Err(unavailable("dashboard"))
}
@@ -183,7 +183,8 @@ async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
let now = Utc::now();
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
active_accounts: 3,
inherited_accounts: 2,
overridden_slots: 5,
used_slots: Some(2),
}));
let services = observability_services(store).await;
@@ -195,9 +196,28 @@ async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
.expect("dashboard summary")
.capacity;
assert_eq!(capacity.total_slots, 3);
assert_eq!(capacity.total_slots, 7);
assert_eq!(capacity.used_slots, Some(2));
assert_eq!(capacity.available_slots, Some(1));
assert_eq!(capacity.available_slots, Some(5));
}
#[tokio::test]
async fn dashboard_summary_should_share_one_current_observation_time_across_runtime_facts() {
let historical_end = Utc::now() - Duration::hours(1);
let store = Arc::new(FixtureObservabilityStore::new(observation_range(
historical_end,
)));
let services = observability_services(store.clone()).await;
services
.observability()
.dashboard_summary(observation_range(historical_end), TrendKind::Usage)
.await
.expect("dashboard summary");
let (summary_observed_at, slots_observed_at) = store.observed_times();
assert_eq!(summary_observed_at, slots_observed_at);
assert!(summary_observed_at.is_some_and(|value| value > historical_end));
}
#[tokio::test]
@@ -394,6 +414,8 @@ struct FixtureObservabilityStore {
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
diagnostics: Mutex<Vec<DiagnosticObservation>>,
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
summary_observed_at: Mutex<Option<DateTime<Utc>>>,
slots_observed_at: Mutex<Option<DateTime<Utc>>>,
usage_records: Mutex<Vec<UsageRecord>>,
}
@@ -410,6 +432,8 @@ impl FixtureObservabilityStore {
calculated_billing_facts: Mutex::new(Vec::new()),
diagnostics: Mutex::new(Vec::new()),
runtime_slots: Mutex::new(None),
summary_observed_at: Mutex::new(None),
slots_observed_at: Mutex::new(None),
usage_records: Mutex::new(Vec::new()),
}
}
@@ -437,6 +461,16 @@ impl FixtureObservabilityStore {
*self.runtime_slots.lock().expect("runtime slots") = runtime_slots;
}
fn observed_times(&self) -> (Option<DateTime<Utc>>, Option<DateTime<Utc>>) {
(
*self
.summary_observed_at
.lock()
.expect("summary observed at"),
*self.slots_observed_at.lock().expect("slots observed at"),
)
}
fn replace_usage_records(&self, records: Vec<UsageRecord>) {
*self.usage_records.lock().expect("usage records") = records;
}
@@ -444,11 +478,20 @@ impl FixtureObservabilityStore {
#[async_trait]
impl ObservabilityStore for FixtureObservabilityStore {
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation> {
async fn dashboard_summary(
&self,
range: TimeRange,
observed_at: DateTime<Utc>,
) -> AdminStoreResult<DashboardObservation> {
*self
.summary_observed_at
.lock()
.expect("summary observed at") = Some(observed_at);
Ok(DashboardObservation {
range,
requests: RequestMetrics::default(),
attempts: AttemptMetrics::default(),
totals: Default::default(),
provider_accounts: AccountPoolMetrics::default(),
trend: self.trend.lock().expect("trend").clone(),
account_usage: Vec::new(),
@@ -458,8 +501,9 @@ impl ObservabilityStore for FixtureObservabilityStore {
async fn dashboard_runtime_slots(
&self,
_: DateTime<Utc>,
observed_at: DateTime<Utc>,
) -> AdminStoreResult<Option<DashboardRuntimeSlots>> {
*self.slots_observed_at.lock().expect("slots observed at") = Some(observed_at);
Ok(*self.runtime_slots.lock().expect("runtime slots"))
}
@@ -611,11 +655,14 @@ fn total_record(
id: id.to_owned(),
client_api_key_ref: "key_billing".to_owned(),
config_revision: 1,
routing_scope: "all".to_owned(),
routing_group_refs: Vec::new(),
routing_group_names_snapshot: Vec::new(),
protocol: "openai".to_owned(),
operation: "generate".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "gpt-5.5".to_owned(),
requested_model_id: Some("gpt-5.5".to_owned()),
provider_kind: provider_kind.map(str::to_owned),
provider_account_ref: None,
provider_account_name: None,
@@ -5,8 +5,8 @@ use gateway_core::engine::credential::ProviderAccountId;
use gateway_admin::{
AdminServices,
model::provider_credentials::{
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, CredentialMutation,
ImportCredentials, ProviderQuotaRequest, StartAuthorization,
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, ImportCredentials,
ProviderQuotaRequest, StartAuthorization,
},
ports::provider::ProviderAdminErrorKind,
};
@@ -15,55 +15,6 @@ use super::accounts::{
FakeAccountStore, FakeProviderAdmin, context, document, events, recorded, revision,
};
#[tokio::test]
async fn openai_enable_should_use_scoped_store_without_provider_mutation() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
let services = service(provider, store.clone()).await;
services
.openai()
.enable(mutation("acct_test"))
.await
.expect("enable credential");
assert_eq!(
recorded(&events),
[
"store.credential_details",
"store.set_enabled",
"provider.account_facts_changed",
]
);
assert_eq!(store.audit_requests(), ["request-openai"]);
}
#[tokio::test]
async fn openai_disable_should_commit_then_release_provider_resources() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
let services = service(provider, store.clone()).await;
services
.openai()
.disable(mutation("acct_test"))
.await
.expect("disable credential");
assert_eq!(
recorded(&events),
[
"store.credential_details",
"store.set_enabled",
"provider.account_unavailable",
"provider.account_facts_changed",
]
);
assert_eq!(store.audit_requests(), ["request-openai"]);
}
#[tokio::test]
async fn openai_delete_should_commit_then_release_provider_resources() {
let events = events();
@@ -89,26 +40,6 @@ async fn openai_delete_should_commit_then_release_provider_resources() {
assert_eq!(store.audit_requests(), ["request-openai"]);
}
#[tokio::test]
async fn openai_disable_store_failure_should_not_release_provider_resources() {
let events = events();
let provider = FakeProviderAdmin::new("openai", events.clone());
let store = FakeAccountStore::new("openai", events.clone());
store.fail_next_commit();
let services = service(provider, store).await;
services
.openai()
.disable(mutation("acct_test"))
.await
.expect_err("failed Store transaction");
assert_eq!(
recorded(&events),
["store.credential_details", "store.set_enabled"]
);
}
#[tokio::test]
async fn openai_import_should_prepare_before_atomic_store_commit() {
let events = events();
@@ -125,6 +56,8 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
.await
.expect("import credential");
await_quota_requests(&provider, 1).await;
assert_eq!(
recorded(&events),
[
@@ -162,6 +95,8 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
.await
.expect("import credentials");
await_quota_requests(&provider, 2).await;
assert_eq!(
result.credential_ids,
[
@@ -203,6 +138,8 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
.await
.expect("committed import remains successful");
await_quota_requests(&provider, 1).await;
assert_eq!(
result.credential_ids,
[ProviderAccountId::new("acct_prepared").expect("account ID")]
@@ -236,6 +173,8 @@ async fn openai_authorization_create_should_observe_initial_quota() {
.await
.expect("complete authorization");
await_quota_requests(&provider, 1).await;
assert_eq!(
recorded(&events),
[
@@ -284,6 +223,8 @@ async fn openai_authorization_create_should_remain_successful_when_initial_quota
.await
.expect("quota observation must not fail authorization");
await_quota_requests(&provider, 1).await;
assert_eq!(result.account_id.as_str(), "acct_prepared");
assert_eq!(provider.quota_requests().len(), 1);
}
@@ -322,6 +263,8 @@ async fn openai_authorization_store_failure_should_release_claim_for_retry() {
.await
.expect("released claim must permit retry");
await_quota_requests(&provider, 1).await;
assert_eq!(
recorded(&events),
[
@@ -400,6 +343,8 @@ async fn openai_reauthorization_should_commit_after_credential_revision_advances
.await
.expect("complete reauthorization");
await_quota_requests(&provider, 1).await;
assert_eq!(result.credential_revision, Some(revision(3)));
assert_eq!(
@@ -426,11 +371,14 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
.await
}
fn mutation(account_id: &str) -> CredentialMutation {
CredentialMutation {
context: context("request-openai"),
account_id: ProviderAccountId::new(account_id).expect("account ID"),
async fn await_quota_requests(provider: &FakeProviderAdmin, expected: usize) {
for _ in 0..100 {
if provider.quota_requests().len() == expected {
return;
}
tokio::task::yield_now().await;
}
assert_eq!(provider.quota_requests().len(), expected);
}
fn deletion(account_id: &str) -> CredentialDeletion {
@@ -4,37 +4,12 @@ use gateway_core::engine::credential::ProviderAccountId;
use gateway_admin::{
AdminServices,
model::provider_credentials::{CredentialMutation, ImportCredentials, ProviderQuotaRequest},
model::provider_credentials::{ImportCredentials, ProviderQuotaRequest},
ports::provider::ProviderAdminErrorKind,
};
use super::accounts::{FakeAccountStore, FakeProviderAdmin, context, document, events, recorded};
#[tokio::test]
async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events.clone());
let services = service(provider.clone(), store.clone()).await;
services
.xai()
.disable(mutation("acct_test"))
.await
.expect("disable credential");
assert_eq!(
recorded(&events),
[
"store.credential_details",
"store.set_enabled",
"provider.account_unavailable",
"provider.account_facts_changed",
]
);
assert_eq!(store.audit_requests(), ["request-xai"]);
}
#[tokio::test]
async fn xai_import_should_prepare_before_atomic_store_commit() {
let events = events();
@@ -132,10 +107,3 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
.build()
.await
}
fn mutation(account_id: &str) -> CredentialMutation {
CredentialMutation {
context: context("request-xai"),
account_id: ProviderAccountId::new(account_id).expect("account ID"),
}
}
@@ -0,0 +1,417 @@
//! Account group HTTP wire and fixed routes.
use std::collections::BTreeMap;
use axum::{
Json, Router,
extract::{Query, State},
http::StatusCode,
response::IntoResponse,
routing::{get, post},
};
use chrono::{DateTime, Utc};
use gateway_admin::model::{
PageSize,
account_groups::{
AccountGroupAccountSummary, AccountGroupCapacity, AccountGroupColor, AccountGroupListQuery,
AccountGroupMutation, AccountGroupPage, AccountGroupRecord, AccountGroupUsage,
CreateAccountGroup, DeleteAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
},
};
use gateway_core::routing::AccountGroupId;
use serde::{Deserialize, Serialize};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError, wire::map_admin_service_error,
};
const DEFAULT_PAGE_SIZE: u32 = 50;
const MAX_PAGE_SIZE: u32 = 200;
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ListAccountGroupsQuery {
page: Option<u32>,
page_size: Option<u32>,
search: Option<String>,
enabled: Option<bool>,
}
impl ListAccountGroupsQuery {
fn into_command(self) -> Result<AccountGroupListQuery, WireValidationError> {
let page = self.page.unwrap_or(1);
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
if page == 0 || page_size == 0 || page_size > MAX_PAGE_SIZE {
return Err(WireValidationError::new("page"));
}
let search = self
.search
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty());
if search
.as_deref()
.is_some_and(|value| value.len() > 256 || value.chars().any(char::is_control))
{
return Err(WireValidationError::new("search"));
}
Ok(AccountGroupListQuery {
page,
page_size: PageSize::new(
u16::try_from(page_size).map_err(|_| WireValidationError::new("pageSize"))?,
)
.map_err(|_| WireValidationError::new("pageSize"))?,
search,
enabled: self.enabled,
})
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct CreateAccountGroupRequest {
name: String,
description: Option<String>,
color: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateAccountGroupRequest {
id: String,
name: String,
description: Option<String>,
color: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct AccountGroupIdRequest {
id: String,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupView {
id: String,
name: String,
description: Option<String>,
color: String,
enabled: bool,
member_count: u64,
provider_counts: BTreeMap<String, u64>,
client_key_count: u64,
account_summary: AccountGroupAccountSummaryView,
capacity: AccountGroupCapacityView,
usage: AccountGroupUsageView,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupAccountSummaryView {
available: u64,
limited: u64,
total: u64,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupCapacityView {
used_slots: Option<u64>,
total_slots: u64,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupUsageView {
today_usd: String,
total_usd: String,
}
impl From<AccountGroupRecord> for AccountGroupView {
fn from(record: AccountGroupRecord) -> Self {
Self {
id: record.id.to_string(),
name: record.name,
description: record.description,
color: record.color.as_str().to_owned(),
enabled: record.enabled,
member_count: record.member_count,
provider_counts: record.provider_counts,
client_key_count: record.client_key_count,
account_summary: account_summary_view(record.account_summary),
capacity: capacity_view(record.capacity),
usage: usage_view(record.usage),
created_at: record.created_at,
updated_at: record.updated_at,
}
}
}
fn account_summary_view(summary: AccountGroupAccountSummary) -> AccountGroupAccountSummaryView {
AccountGroupAccountSummaryView {
available: summary.available,
limited: summary.limited,
total: summary.total,
}
}
fn capacity_view(capacity: AccountGroupCapacity) -> AccountGroupCapacityView {
AccountGroupCapacityView {
used_slots: capacity.used_slots,
total_slots: capacity.total_slots,
}
}
fn usage_view(usage: AccountGroupUsage) -> AccountGroupUsageView {
AccountGroupUsageView {
today_usd: usage.today_usd.to_string(),
total_usd: usage.total_usd.to_string(),
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupPageData {
items: Vec<AccountGroupView>,
page: PageMeta,
config_revision: u64,
}
impl From<AccountGroupPage> for AccountGroupPageData {
fn from(page: AccountGroupPage) -> Self {
let total_pages = if page.total == 0 {
0
} else {
page.total.div_ceil(u64::from(page.page_size))
};
Self {
items: page.items.into_iter().map(Into::into).collect(),
page: PageMeta::new(
page.page,
u32::from(page.page_size),
page.total,
u32::try_from(total_pages).unwrap_or(u32::MAX),
),
config_revision: page.config_revision.get(),
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountGroupMutationData {
id: String,
record: Option<AccountGroupView>,
config_revision: u64,
}
impl From<AccountGroupMutation> for AccountGroupMutationData {
fn from(mutation: AccountGroupMutation) -> Self {
Self {
id: mutation.id.to_string(),
record: mutation.record.map(Into::into),
config_revision: mutation.config_revision.get(),
}
}
}
/// Construct all fixed account-group management routes.
pub fn router<S>() -> Router<S>
where
S: AdminSessionState + Clone + Send + Sync + 'static,
{
Router::new()
.route("/api/admin/account-groups", get(list::<S>))
.route("/api/admin/account-groups/create", post(create::<S>))
.route("/api/admin/account-groups/update", post(update::<S>))
.route("/api/admin/account-groups/enable", post(enable::<S>))
.route("/api/admin/account-groups/disable", post(disable::<S>))
.route("/api/admin/account-groups/delete", post(delete::<S>))
}
async fn list<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<ListAccountGroupsQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let result = state
.admin_services()
.account_groups()
.list(query.into_command().map_err(map_wire_error)?)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountGroupPageData::from(result)),
))
}
async fn create<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<CreateAccountGroupRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
validate_group_fields(&request.name, request.description.as_deref())?;
mutation_response(
StatusCode::CREATED,
state
.admin_services()
.account_groups()
.create(
&auth.context().mutation_context(),
CreateAccountGroup {
name: request.name,
description: request.description,
color: group_color(&request.color)?,
},
)
.await,
)
}
async fn update<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<UpdateAccountGroupRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
validate_group_fields(&request.name, request.description.as_deref())?;
mutation_response(
StatusCode::OK,
state
.admin_services()
.account_groups()
.update(
&auth.context().mutation_context(),
UpdateAccountGroup {
id: group_id(request.id)?,
name: request.name,
description: request.description,
color: group_color(&request.color)?,
},
)
.await,
)
}
async fn enable<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountGroupIdRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
set_enabled(auth, state, request, true).await
}
async fn disable<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountGroupIdRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
set_enabled(auth, state, request, false).await
}
async fn set_enabled<S>(
auth: AdminAuth,
state: S,
request: AccountGroupIdRequest,
enabled: bool,
) -> Result<AdminResponse<AdminEnvelope<AccountGroupMutationData>>, AdminError>
where
S: AdminSessionState + Send + Sync,
{
mutation_response(
StatusCode::OK,
state
.admin_services()
.account_groups()
.set_enabled(
&auth.context().mutation_context(),
SetAccountGroupEnabled {
id: group_id(request.id)?,
enabled,
},
)
.await,
)
}
async fn delete<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountGroupIdRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
mutation_response(
StatusCode::OK,
state
.admin_services()
.account_groups()
.delete(
&auth.context().mutation_context(),
DeleteAccountGroup {
id: group_id(request.id)?,
},
)
.await,
)
}
fn mutation_response(
status: StatusCode,
result: Result<AccountGroupMutation, gateway_admin::model::AdminError>,
) -> Result<AdminResponse<AdminEnvelope<AccountGroupMutationData>>, AdminError> {
let result = result.map_err(map_service_error)?;
Ok(AdminResponse::new(
status,
AdminEnvelope::ok(AccountGroupMutationData::from(result)),
))
}
fn group_id(value: String) -> Result<AccountGroupId, AdminError> {
AccountGroupId::new(value).map_err(|_| AdminError::bad_request("Invalid account group ID"))
}
fn group_color(value: &str) -> Result<AccountGroupColor, AdminError> {
AccountGroupColor::parse(value)
.ok_or_else(|| AdminError::bad_request("Invalid account group color"))
}
fn validate_group_fields(name: &str, description: Option<&str>) -> Result<(), AdminError> {
if name.trim() != name
|| name.is_empty()
|| name.chars().count() > 100
|| name.chars().any(char::is_control)
|| description
.is_some_and(|value| value.len() > 4096 || value.chars().any(char::is_control))
{
return Err(AdminError::bad_request("Invalid account group request"));
}
Ok(())
}
fn map_wire_error(_: WireValidationError) -> AdminError {
AdminError::bad_request("Invalid account group query")
}
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
map_admin_service_error(error, "Account group repository unavailable")
}
+446 -107
View File
@@ -18,26 +18,29 @@ use futures::{Stream, StreamExt as _};
use gateway_admin::model::{
AdminError as AdminServiceError, PageSize,
accounts::{
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
AccountUsage, SortDirection,
AccountConcurrencyLimit, AccountConnectionTestEvent as DomainConnectionTestEvent,
AccountCost, AccountGroupFilter, AccountListQuery, AccountModelUsage, AccountSort,
AccountSortField, AccountStatus as DomainAccountStatus, AccountUpdateResult, AccountUsage,
AccountWeight, AccountsUpdateResult, BatchUpdateAccounts, SortDirection, UpdateAccount,
},
provider_credentials::{
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
AuthorizationStarted, CompleteAuthorization, CredentialDeletion, CredentialDeletionResult,
CredentialImportResult, CredentialMutation, CredentialMutationResult, ImportCredentials,
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow, RotateCredential,
StartAuthorization,
AuthorizationStarted, CompleteAuthorization, ConsumeProviderResetCredit,
CredentialDeletion, CredentialDeletionResult, CredentialImportResult, CredentialMutation,
CredentialMutationResult, ImportCredentials, ProviderDocument, ProviderModels,
ProviderQuota, ProviderQuotaWindow, ProviderResetCredit, ProviderResetCreditResult,
ProviderResetCredits, RotateCredential, StartAuthorization,
},
};
use gateway_core::{
engine::credential::{OpaqueProviderData, ProviderAccountId},
routing::{ProviderKind, UpstreamModelId},
routing::{AccountGroupId, ProviderKind, UpstreamModelId},
};
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{Map, Value};
use uuid::{Uuid, Version};
use super::presenter::{format_compact_number, format_number};
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError,
@@ -51,8 +54,10 @@ const MAX_NAME_BYTES: usize = 512;
const MAX_IMPORT_DATA_BYTES: usize = 64 * 1024 * 1024;
const MAX_ACCESS_TOKEN_BYTES: usize = 16 * 1024;
const MAX_REFRESH_TOKEN_BYTES: usize = 64 * 1024;
const MAX_ID_TOKEN_BYTES: usize = 16 * 1024;
const MAX_CALLBACK_URL_BYTES: usize = 64 * 1024;
const MAX_ACCOUNT_DELETE_BATCH: usize = 200;
const MAX_ACCOUNT_GROUP_BATCH: usize = 1000;
/// 账号列表查询参数。
#[derive(Debug, Clone, Default, Deserialize)]
@@ -61,12 +66,74 @@ pub struct ListQuery {
pub page: Option<u32>,
pub page_size: Option<u32>,
pub provider: Option<String>,
pub group_id: Option<String>,
pub search: Option<String>,
pub status: Option<String>,
pub sort_by: Option<String>,
pub sort_direction: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BatchUpdateAccountsRequest {
pub account_ids: Vec<String>,
pub enabled: bool,
#[serde(deserialize_with = "deserialize_required_nullable")]
pub concurrency_limit: Option<u64>,
pub weight: u64,
pub group_ids: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct BatchUpdatedAccountsData {
account_ids: Vec<String>,
config_revision: u64,
}
impl From<AccountsUpdateResult> for BatchUpdatedAccountsData {
fn from(result: AccountsUpdateResult) -> Self {
Self {
account_ids: result
.account_ids
.into_iter()
.map(|id| id.to_string())
.collect(),
config_revision: result.config_revision.get(),
}
}
}
impl BatchUpdateAccountsRequest {
pub fn validate(&self) -> Result<(), WireValidationError> {
if self.account_ids.is_empty()
|| self.account_ids.len() > MAX_ACCOUNT_GROUP_BATCH
|| self
.account_ids
.iter()
.any(|id| require_account_id(id, "accountIds").is_err())
|| self.account_ids.iter().collect::<BTreeSet<_>>().len() != self.account_ids.len()
{
return Err(WireValidationError::new("accountIds"));
}
validate_wire_group_ids(&self.group_ids)?;
parse_concurrency_limit(self.concurrency_limit)?;
parse_account_weight(self.weight)?;
Ok(())
}
fn into_command(self) -> Result<BatchUpdateAccounts, WireValidationError> {
self.validate()?;
Ok(BatchUpdateAccounts {
account_ids: self.account_ids,
enabled: self.enabled,
concurrency_limit: parse_concurrency_limit(self.concurrency_limit)?,
weight: parse_account_weight(self.weight)?,
group_ids: validate_wire_group_ids(&self.group_ids)?,
})
}
}
impl ListQuery {
/// 解析并校验全部 wire 字段,生成 Admin 查询命令。
pub fn validate(self) -> Result<AccountListQuery, WireValidationError> {
@@ -79,6 +146,14 @@ impl ListQuery {
return Err(WireValidationError::new("pageSize"));
}
let provider_kind = parse_provider(self.provider.as_deref().unwrap_or("all"))?;
let group_filter = match self.group_id.as_deref().map(str::trim) {
None | Some("") => None,
Some("ungrouped") => Some(AccountGroupFilter::Ungrouped),
Some(value) => Some(AccountGroupFilter::Group(
AccountGroupId::new(value.to_owned())
.map_err(|_| WireValidationError::new("groupId"))?,
)),
};
let search = self
.search
.map(|value| value.trim().to_owned())
@@ -111,6 +186,7 @@ impl ListQuery {
)
.map_err(|_| WireValidationError::new("pageSize"))?,
provider_kind,
group_filter,
search,
status,
sort,
@@ -176,6 +252,7 @@ pub struct AccountView {
pub id: String,
pub name: String,
pub provider: String,
pub groups: Vec<AccountGroupRefView>,
pub resource_ref: String,
pub email: Option<String>,
pub account_id: Option<String>,
@@ -185,13 +262,13 @@ pub struct AccountView {
pub authentication_kind: String,
pub has_refresh_token: bool,
pub status: String,
pub display_status: String,
/// `status == "error"` 时的具体原因;其余状态为 `null`。
pub error_reason: Option<String>,
/// 最近一次失败的上游错误描述;仅错误状态存在。
pub error_message: Option<String>,
pub availability: String,
pub enabled: bool,
pub concurrency_limit: Option<u32>,
pub weight: u16,
pub access_token_expires_at: Option<String>,
pub access_token_expires_at_display: Option<String>,
pub refresh_token_expires_at: Option<String>,
@@ -204,6 +281,15 @@ pub struct AccountView {
pub usage: AccountUsageView,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountGroupRefView {
pub id: String,
pub name: String,
pub color: String,
pub enabled: bool,
}
/// Provider quota 安全视图。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
@@ -221,8 +307,12 @@ pub struct AccountQuotaView {
pub struct AccountQuotaWindowView {
pub key: String,
pub group: String,
pub limit_id: Option<String>,
pub limit_name: Option<String>,
pub role: Option<String>,
pub window_seconds: Option<u64>,
pub label_display: String,
pub window_label_display: String,
pub used_percent: Option<f64>,
pub used_percent_display: String,
pub limit_reached: bool,
@@ -388,6 +478,48 @@ impl AccountActionRequest {
}
}
/// 主动额度重置卡消费请求。幂等键由 UI 生成并在不确定重试时复用,与官方一致。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AccountResetCreditConsumeRequest {
pub account_id: String,
#[serde(default)]
pub credit_id: Option<String>,
pub redeem_request_id: String,
}
impl AccountResetCreditConsumeRequest {
pub fn validate(&self) -> Result<(), WireValidationError> {
require_account_id(&self.account_id, "accountId")?;
if self.credit_id.as_deref().is_some_and(|value| {
let value = value.trim();
value.is_empty() || value.len() > 512 || value.chars().any(char::is_control)
}) {
return Err(WireValidationError::new("creditId"));
}
let value = self.redeem_request_id.trim();
let redeem_request_id =
Uuid::parse_str(value).map_err(|_| WireValidationError::new("redeemRequestId"))?;
if redeem_request_id.get_version() != Some(Version::Random)
|| redeem_request_id.hyphenated().to_string() != value
{
return Err(WireValidationError::new("redeemRequestId"));
}
Ok(())
}
fn into_command(self) -> Result<ConsumeProviderResetCredit, WireValidationError> {
self.validate()?;
Ok(ConsumeProviderResetCredit {
account_id: ProviderAccountId::new(self.account_id)
.map_err(|_| WireValidationError::new("accountId"))?,
credit_id: self.credit_id.map(|value| value.trim().to_owned()),
redeem_request_id: Uuid::parse_str(&self.redeem_request_id)
.map_err(|_| WireValidationError::new("redeemRequestId"))?,
})
}
}
/// 手工 OAuth 刷新会变更持久 credential。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -455,6 +587,67 @@ pub struct AccountQuotaData {
pub account: AccountView,
}
/// 主动额度重置卡列表响应。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountResetCreditsData {
pub available_count: u64,
pub credits: Vec<AccountResetCreditView>,
}
/// 一张安全主动额度重置卡视图。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountResetCreditView {
pub id: String,
pub status: Option<String>,
pub title: Option<String>,
pub expires_at: Option<String>,
pub reset_type: Option<String>,
}
/// 主动额度重置卡消费响应。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountResetCreditResultData {
pub code: String,
pub credit: Option<AccountResetCreditView>,
}
impl From<ProviderResetCredit> for AccountResetCreditView {
fn from(credit: ProviderResetCredit) -> Self {
Self {
id: credit.id,
status: credit.status,
title: credit.title,
expires_at: credit.expires_at.map(|value| value.to_rfc3339()),
reset_type: credit.reset_type,
}
}
}
impl From<ProviderResetCredits> for AccountResetCreditsData {
fn from(credits: ProviderResetCredits) -> Self {
Self {
available_count: credits.available_count,
credits: credits
.credits
.into_iter()
.map(AccountResetCreditView::from)
.collect(),
}
}
}
impl From<ProviderResetCreditResult> for AccountResetCreditResultData {
fn from(result: ProviderResetCreditResult) -> Self {
Self {
code: result.code,
credit: result.credit.map(AccountResetCreditView::from),
}
}
}
/// Provider-owned 明文导出文档;Debug 永远不输出内部 JSON。
#[derive(Serialize)]
#[serde(transparent)]
@@ -522,10 +715,9 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
DomainConnectionTestEvent::Content { text } => {
serde_json::json!({ "type": "content", "text": text })
}
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
DomainConnectionTestEvent::Completed => serde_json::json!({
"type": "test_complete",
"success": true,
"accountStatus": account_status.as_str()
"success": true
}),
DomainConnectionTestEvent::Failed {
message,
@@ -534,7 +726,6 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
upstream_status,
upstream_content_type,
upstream_body,
account_status,
} => serde_json::json!({
"type": "error",
"error": message,
@@ -542,8 +733,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
"providerErrorType": provider_error_type,
"upstreamStatus": upstream_status,
"upstreamContentType": upstream_content_type,
"upstreamBody": upstream_body,
"accountStatus": account_status.as_str()
"upstreamBody": upstream_body
}),
};
Self { data }
@@ -686,31 +876,49 @@ impl CompleteAccountAuthorizationRequest {
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AccountMutationRequest {
pub provider: String,
pub struct UpdateAccountRequest {
pub account_id: String,
pub enabled: bool,
#[serde(deserialize_with = "deserialize_required_nullable")]
pub concurrency_limit: Option<u64>,
pub weight: u64,
pub group_ids: Vec<String>,
}
impl AccountMutationRequest {
impl UpdateAccountRequest {
pub fn validate(&self) -> Result<(), WireValidationError> {
AccountProvider::parse(&self.provider)?;
require_account_id(&self.account_id, "accountId")
require_account_id(&self.account_id, "accountId")?;
parse_concurrency_limit(self.concurrency_limit)?;
parse_account_weight(self.weight)?;
validate_wire_group_ids(&self.group_ids)?;
Ok(())
}
fn into_command(
self,
context: gateway_admin::model::MutationContext,
) -> Result<(AccountProvider, CredentialMutation), WireValidationError> {
fn into_command(self) -> Result<UpdateAccount, WireValidationError> {
self.validate()?;
let provider = AccountProvider::parse(&self.provider)?;
Ok((
provider,
CredentialMutation {
context,
account_id: ProviderAccountId::new(self.account_id)
.map_err(|_| WireValidationError::new("accountId"))?,
},
))
Ok(UpdateAccount {
account_id: self.account_id,
enabled: self.enabled,
concurrency_limit: parse_concurrency_limit(self.concurrency_limit)?,
weight: parse_account_weight(self.weight)?,
group_ids: validate_wire_group_ids(&self.group_ids)?,
})
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UpdatedAccountData {
pub account_id: String,
pub config_revision: u64,
}
impl From<AccountUpdateResult> for UpdatedAccountData {
fn from(result: AccountUpdateResult) -> Self {
Self {
account_id: result.account_id.to_string(),
config_revision: result.config_revision.get(),
}
}
}
@@ -765,6 +973,7 @@ pub struct RotateAccountRequest {
pub account_id: String,
pub access_token: String,
pub refresh_token: Option<String>,
pub id_token: Option<String>,
}
impl RotateAccountRequest {
@@ -773,7 +982,11 @@ impl RotateAccountRequest {
return Err(WireValidationError::new("provider"));
}
require_account_id(&self.account_id, "accountId")?;
validate_oauth_material(&self.access_token, self.refresh_token.as_deref())
validate_oauth_material(
&self.access_token,
self.refresh_token.as_deref(),
self.id_token.as_deref(),
)
}
fn into_command(
@@ -787,6 +1000,10 @@ impl RotateAccountRequest {
"refresh_token".to_owned(),
self.refresh_token.map_or(Value::Null, Value::String),
);
material.insert(
"id_token".to_owned(),
self.id_token.map_or(Value::Null, Value::String),
);
Ok(RotateCredential {
mutation: CredentialMutation {
context,
@@ -805,8 +1022,8 @@ pub struct AccountImportData {
pub account_ids: Vec<String>,
}
impl From<CredentialImportResult> for AccountImportData {
fn from(result: CredentialImportResult) -> Self {
impl AccountImportData {
pub fn from_result(result: CredentialImportResult) -> Self {
let account_ids = result
.credential_ids
.into_iter()
@@ -883,6 +1100,46 @@ fn require_account_id(value: &str, field: &'static str) -> Result<(), WireValida
Ok(())
}
fn parse_concurrency_limit(
value: Option<u64>,
) -> Result<Option<AccountConcurrencyLimit>, WireValidationError> {
value
.map(|value| {
u32::try_from(value)
.ok()
.and_then(AccountConcurrencyLimit::new)
.ok_or_else(|| WireValidationError::new("concurrencyLimit"))
})
.transpose()
}
fn deserialize_required_nullable<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
where
D: Deserializer<'de>,
{
Option::<u64>::deserialize(deserializer)
}
fn parse_account_weight(value: u64) -> Result<AccountWeight, WireValidationError> {
u16::try_from(value)
.ok()
.and_then(AccountWeight::new)
.ok_or_else(|| WireValidationError::new("weight"))
}
fn validate_wire_group_ids(values: &[String]) -> Result<Vec<AccountGroupId>, WireValidationError> {
if values.len() > MAX_ACCOUNT_GROUP_BATCH
|| values.iter().collect::<BTreeSet<_>>().len() != values.len()
{
return Err(WireValidationError::new("groupIds"));
}
values
.iter()
.cloned()
.map(|value| AccountGroupId::new(value).map_err(|_| WireValidationError::new("groupIds")))
.collect()
}
/// 构造统一账号管理路由。
pub fn router<S>() -> Router<S>
where
@@ -894,11 +1151,19 @@ where
.route("/api/admin/accounts/export", get(export_accounts::<S>))
.route("/api/admin/accounts/import", post(import_accounts::<S>))
.route("/api/admin/accounts/refresh", post(refresh_account::<S>))
.route("/api/admin/accounts/recover", post(recover_account::<S>))
.route("/api/admin/accounts/rotate", post(rotate_account::<S>))
.route("/api/admin/accounts/enable", post(enable_account::<S>))
.route("/api/admin/accounts/disable", post(disable_account::<S>))
.route("/api/admin/accounts/update", post(update_account::<S>))
.route("/api/admin/accounts/delete", post(delete_accounts::<S>))
.route(
"/api/admin/accounts/batch-update",
post(batch_update_accounts::<S>),
)
.route("/api/admin/accounts/quota", get(account_quota::<S>))
.route(
"/api/admin/accounts/reset-credits",
get(account_reset_credits::<S>).post(consume_account_reset_credit::<S>),
)
.route(
"/api/admin/accounts/quota/refresh",
post(refresh_account_quota::<S>),
@@ -922,6 +1187,27 @@ where
)
}
async fn batch_update_accounts<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<BatchUpdateAccountsRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let command = request.into_command().map_err(map_wire_error)?;
let result = state
.admin_services()
.accounts()
.batch_update(&auth.context().mutation_context(), command)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BatchUpdatedAccountsData::from(result)),
))
}
async fn list_accounts<S>(
_auth: AdminAuth,
State(state): State<S>,
@@ -1011,7 +1297,7 @@ where
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::CREATED,
AdminEnvelope::ok(AccountImportData::from(result)),
AdminEnvelope::ok(AccountImportData::from_result(result)),
))
}
@@ -1106,64 +1392,24 @@ where
))
}
async fn enable_account<S>(
async fn update_account<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountMutationRequest>,
Json(request): Json<UpdateAccountRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
mutate_account(auth, state, request, AccountMutationAction::Enable).await
}
async fn disable_account<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountMutationRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
mutate_account(auth, state, request, AccountMutationAction::Disable).await
}
#[derive(Clone, Copy)]
enum AccountMutationAction {
Enable,
Disable,
}
async fn mutate_account<S>(
auth: AdminAuth,
state: S,
request: AccountMutationRequest,
action: AccountMutationAction,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let (provider, command) = request
.into_command(auth.context().mutation_context())
.map_err(map_wire_error)?;
let result = match (provider, action) {
(AccountProvider::OpenAi, AccountMutationAction::Enable) => {
state.admin_services().openai().enable(command).await
}
(AccountProvider::OpenAi, AccountMutationAction::Disable) => {
state.admin_services().openai().disable(command).await
}
(AccountProvider::Xai, AccountMutationAction::Enable) => {
state.admin_services().xai().enable(command).await
}
(AccountProvider::Xai, AccountMutationAction::Disable) => {
state.admin_services().xai().disable(command).await
}
}
.map_err(map_service_error)?;
let command = request.into_command().map_err(map_wire_error)?;
let result = state
.admin_services()
.accounts()
.update(&auth.context().mutation_context(), command)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountMutationData::from(result)),
AdminEnvelope::ok(UpdatedAccountData::from(result)),
))
}
@@ -1208,6 +1454,25 @@ where
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
}
async fn recover_account<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountActionRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let account_id = request.into_id().map_err(map_wire_error)?;
let result = state
.admin_services()
.accounts()
.recover(&auth.context().mutation_context(), account_id)
.await
.map_err(map_service_error)?;
let data = account_refresh_data(result, Utc::now());
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
}
async fn account_quota<S>(
_auth: AdminAuth,
State(state): State<S>,
@@ -1250,6 +1515,48 @@ where
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
}
async fn account_reset_credits<S>(
auth: AdminAuth,
State(state): State<S>,
Query(query): Query<AccountIdQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let account_id = query.into_id().map_err(map_wire_error)?;
let result = state
.admin_services()
.accounts()
.reset_credits(&auth.context().mutation_context(), account_id)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountResetCreditsData::from(result)),
))
}
async fn consume_account_reset_credit<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<AccountResetCreditConsumeRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let command = request.into_command().map_err(map_wire_error)?;
let result = state
.admin_services()
.accounts()
.consume_reset_credit(&auth.context().mutation_context(), command)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountResetCreditResultData::from(result)),
))
}
async fn account_models<S>(
_auth: AdminAuth,
State(state): State<S>,
@@ -1365,34 +1672,46 @@ fn account_models_data(result: ProviderModels) -> AccountModelsData {
fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
let AccountDirectoryItem {
account,
status,
error_reason,
projection,
usage,
quota,
} = item;
let status = status.as_str().to_owned();
let status = projection.status.as_str().to_owned();
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
let added_at = china_rfc3339(&account.created_at);
let updated_at = china_rfc3339(&account.updated_at);
let (quota, refresh_token_expires_at) = account_quota_view(quota, now);
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
AccountView {
id: account.id.clone(),
name: account.name,
provider: account.provider_kind.to_string(),
groups: account
.groups
.into_iter()
.map(|group| AccountGroupRefView {
id: group.id.to_string(),
name: group.name,
color: group.color.as_str().to_owned(),
enabled: group.enabled,
})
.collect(),
resource_ref: account.id,
email: account.email,
account_id: account.upstream_account_id,
user_id: Some(account.upstream_user_id),
user_id: account.upstream_user_id,
label: None,
plan_type: account.plan_type,
authentication_kind: account.authentication_kind,
has_refresh_token: account.has_refresh_token,
status: status.clone(),
display_status: status,
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
error_message: account.last_error_message,
availability: account.availability.as_str().to_owned(),
status,
error_reason: projection
.error_reason
.map(|reason| reason.as_str().to_owned()),
error_message: projection.error_message,
enabled: account.enabled,
concurrency_limit: account.concurrency_limit.map(|limit| limit.get()),
weight: account.weight.get(),
access_token_expires_at: expires_at,
access_token_expires_at_display: account
.access_token_expires_at
@@ -1410,9 +1729,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
}
fn account_quota_view(
quota: ProviderQuota,
mut quota: ProviderQuota,
rate_limited_until: Option<DateTime<Utc>>,
now: DateTime<Utc>,
) -> (AccountQuotaView, Option<String>) {
quota.apply_limit_reached_display();
let refresh_token_expires_at = quota
.refresh_token_expires_at
.map(|value| china_rfc3339(&value));
@@ -1420,7 +1741,7 @@ fn account_quota_view(
.observed_at
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
let windows = quota.windows.into_iter().map(quota_window_view).collect();
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
(
AccountQuotaView {
refreshed_at_display,
@@ -1437,7 +1758,10 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
key,
group,
label,
source: _,
limit_id,
limit_name,
role,
local_usage_attribution: _,
window_seconds,
used_percent,
reset_at,
@@ -1445,10 +1769,17 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
local_usage,
provider_data: _,
} = window;
let label_display = limit_name
.as_ref()
.map_or_else(|| label.clone(), |name| format!("{name} · {label}"));
AccountQuotaWindowView {
label_display: label,
label_display,
window_label_display: label,
key,
group,
limit_id,
limit_name,
role: role.map(|value| value.as_str().to_owned()),
window_seconds,
used_percent,
used_percent_display: used_percent
@@ -1589,7 +1920,7 @@ fn account_model_usage_view(usage: AccountModelUsage, now: DateTime<Utc>) -> Mod
billing_amount_usd: usd.map(|cost| cost.amount.as_str().to_owned()),
billing_amount_usd_display: usd.map_or_else(
|| "—".to_owned(),
|cost| format!("${}", cost.amount.as_str()),
|cost| format_decimal_currency(cost.amount.as_str(), "USD"),
),
cost_estimate_status: cost_estimate_status.to_owned(),
known_cost_count: known_count,
@@ -1684,6 +2015,7 @@ fn require_text(
fn validate_oauth_material(
access_token: &str,
refresh_token: Option<&str>,
id_token: Option<&str>,
) -> Result<(), WireValidationError> {
if access_token.len() > MAX_ACCESS_TOKEN_BYTES
|| !valid_visible_ascii(access_token)
@@ -1698,6 +2030,13 @@ fn validate_oauth_material(
}) {
return Err(WireValidationError::new("refreshToken"));
}
if id_token.is_some_and(|token| {
token.len() > MAX_ID_TOKEN_BYTES
|| !valid_visible_ascii(token)
|| !valid_compact_jwt_shape(token)
}) {
return Err(WireValidationError::new("idToken"));
}
Ok(())
}
@@ -12,7 +12,7 @@ use gateway_admin::model::client_keys::{
};
use gateway_core::{
policy::{ClientApiKeyId, RateLimits},
routing::ProviderKind,
routing::AccountGroupId,
};
use serde::{Deserialize, Serialize};
@@ -137,7 +137,7 @@ impl ClientKeySort {
pub struct CreateClientKeyRequest {
name: String,
label: Option<String>,
provider_kind: String,
group_ids: Vec<String>,
max_concurrency: u64,
requests_per_minute: u64,
}
@@ -147,14 +147,13 @@ impl CreateClientKeyRequest {
pub fn into_command(self) -> Result<CreateClientKey, WireValidationError> {
validate_required_text(&self.name, "name")?;
validate_optional_text(self.label.as_deref(), "label")?;
validate_provider_kind(&self.provider_kind)?;
let group_ids = validate_group_ids(self.group_ids)?;
validate_limit(self.max_concurrency, "maxConcurrency")?;
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
Ok(CreateClientKey {
name: self.name,
label: self.label,
provider_kind: ProviderKind::new(self.provider_kind)
.map_err(|_| WireValidationError::new("providerKind"))?,
group_ids,
limits: RateLimits {
max_concurrency: self.max_concurrency,
requests_per_minute: self.requests_per_minute,
@@ -170,7 +169,7 @@ pub struct UpdateClientKeyRequest {
id: String,
name: String,
label: Option<String>,
provider_kind: String,
group_ids: Vec<String>,
max_concurrency: u64,
requests_per_minute: u64,
}
@@ -181,15 +180,14 @@ impl UpdateClientKeyRequest {
validate_required_text(&self.id, "id")?;
validate_required_text(&self.name, "name")?;
validate_optional_text(self.label.as_deref(), "label")?;
validate_provider_kind(&self.provider_kind)?;
let group_ids = validate_group_ids(self.group_ids)?;
validate_limit(self.max_concurrency, "maxConcurrency")?;
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
Ok(UpdateClientKey {
id: client_key_id(self.id, "clientKeyMutationNotFound")?,
name: self.name,
label: self.label,
provider_kind: ProviderKind::new(self.provider_kind)
.map_err(|_| WireValidationError::new("providerKind"))?,
group_ids,
limits: RateLimits {
max_concurrency: self.max_concurrency,
requests_per_minute: self.requests_per_minute,
@@ -242,7 +240,9 @@ pub struct ClientKeyView {
id: String,
name: String,
label: Option<String>,
provider_kind: String,
routing_scope: &'static str,
groups: Vec<ClientKeyGroupView>,
provider_kinds: Vec<String>,
prefix: String,
enabled: bool,
max_concurrency: u64,
@@ -252,13 +252,42 @@ pub struct ClientKeyView {
last_used_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ClientKeyGroupView {
id: String,
name: String,
color: String,
enabled: bool,
}
impl From<ClientKeyRecord> for ClientKeyView {
fn from(record: ClientKeyRecord) -> Self {
let routing_scope = if record.groups.is_empty() {
"all"
} else {
"groups"
};
Self {
id: record.id.to_string(),
name: record.name,
label: record.label,
provider_kind: record.provider_kind.to_string(),
routing_scope,
groups: record
.groups
.into_iter()
.map(|group| ClientKeyGroupView {
id: group.id.to_string(),
name: group.name,
color: group.color.as_str().to_owned(),
enabled: group.enabled,
})
.collect(),
provider_kinds: record
.provider_kinds
.into_iter()
.map(|provider| provider.to_string())
.collect(),
prefix: record.prefix,
enabled: record.enabled,
max_concurrency: record.limits.max_concurrency,
@@ -572,21 +601,20 @@ fn validate_required_text(value: &str, field: &'static str) -> Result<(), WireVa
Ok(())
}
fn validate_provider_kind(value: &str) -> Result<(), WireValidationError> {
let valid = !value.is_empty()
&& value.len() <= 64
&& value.bytes().enumerate().all(|(index, byte)| {
byte.is_ascii_lowercase()
|| byte.is_ascii_digit()
|| (byte == b'-' && index > 0 && index + 1 < value.len())
})
&& !value.starts_with('-')
&& !value.ends_with('-');
if valid {
Ok(())
} else {
Err(WireValidationError::new("providerKind"))
fn validate_group_ids(values: Vec<String>) -> Result<Vec<AccountGroupId>, WireValidationError> {
if values.len() > 1000
|| values
.iter()
.collect::<std::collections::BTreeSet<_>>()
.len()
!= values.len()
{
return Err(WireValidationError::new("groupIds"));
}
values
.into_iter()
.map(|value| AccountGroupId::new(value).map_err(|_| WireValidationError::new("groupIds")))
.collect()
}
fn validate_optional_text(
@@ -7,6 +7,7 @@ use axum::{
response::Response,
};
pub mod account_groups;
pub mod accounts;
pub mod auth;
pub mod backups;
@@ -29,6 +30,7 @@ where
S: AdminSessionState + Clone + Send + Sync + 'static,
{
Router::new()
.merge(account_groups::router::<S>())
.merge(accounts::router::<S>())
.merge(auth::router::<S>())
.merge(backups::router::<S>())
@@ -15,7 +15,7 @@ use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::presenter::{format_compact_number, format_number};
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError, wire::map_admin_service_error,
@@ -127,20 +127,6 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
}
}
pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String {
if currency == "USD" {
let value = amount.parse::<f64>().map_or(0.0, |value| value);
let precision = if value != 0.0 && value.abs() < 0.01 {
4
} else {
2
};
format!("${value:.precision$}")
} else {
format!("{currency} {amount}")
}
}
pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String {
format_decimal_currency(cost.amount.as_str(), &cost.currency)
}
@@ -236,7 +222,7 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
let model = record
.upstream_model_id
.clone()
.unwrap_or_else(|| record.requested_model_id.clone());
.or_else(|| record.requested_model_id.clone());
let transport = record
.upstream_transport
.clone()
@@ -246,6 +232,9 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
id: record.id.clone(),
request_id: record.id,
client_api_key_id: Some(record.client_api_key_ref),
routing_scope: record.routing_scope,
routing_group_refs: record.routing_group_refs,
routing_group_names_snapshot: record.routing_group_names_snapshot,
kind: record.operation,
provider: record.provider_kind,
authentication_kind: record.provider_account_authentication_kind,
@@ -254,7 +243,7 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
account_name: record.provider_account_name,
route: record.endpoint,
model,
requested_model: Some(record.requested_model_id),
requested_model: record.requested_model_id,
upstream_model: record.upstream_model_id,
service_tier: record.service_tier,
status_code,
@@ -362,9 +351,7 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV
provider: attempt
.provider_kind
.unwrap_or_else(|| "unknown".to_owned()),
model: attempt
.upstream_model_id
.unwrap_or_else(|| "unknown".to_owned()),
model: attempt.upstream_model_id,
transport: attempt
.upstream_transport
.unwrap_or_else(|| "unknown".to_owned()),
@@ -709,7 +696,7 @@ pub(crate) fn dashboard_view(
} = result;
let domain::DashboardObservation {
range,
requests,
totals,
provider_accounts,
account_usage,
recent_requests,
@@ -742,27 +729,31 @@ pub(crate) fn dashboard_view(
last_used: relative_time(credential.last_used_at, range.end),
});
}
let unavailable_accounts = provider_accounts
.total
.saturating_sub(provider_accounts.normal);
DashboardDataView {
cards: DashboardCardsView {
credentials: DashboardCredentialsCardView {
total: format_compact_number(provider_accounts.total),
total: provider_accounts.total.to_string(),
total_value: provider_accounts.total,
available: format_compact_number(provider_accounts.active),
available_value: provider_accounts.active,
unavailable: format_compact_number(provider_accounts.unavailable),
unavailable_value: provider_accounts.unavailable,
available: provider_accounts.normal.to_string(),
available_value: provider_accounts.normal,
unavailable: unavailable_accounts.to_string(),
unavailable_value: unavailable_accounts,
},
traffic: DashboardTrafficCardView {
today_requests: format_compact_number(today.request_count),
today_requests_value: today.request_count,
yesterday_requests_value: yesterday.request_count,
total_requests: format_compact_number(requests.request_count),
total_requests: format_compact_number(totals.request_count),
},
tokens: DashboardTokensCardView {
today_tokens: format_compact_number(today.total_tokens),
today_tokens_value: today.total_tokens,
yesterday_tokens_value: yesterday.total_tokens,
total_tokens: format_compact_number(requests.total_tokens),
total_tokens: format_compact_number(totals.total_tokens),
total_billing_amount_usd: total_billing_usd
.as_ref()
.map_or_else(|| "—".to_owned(), |amount| format!("${}", amount.as_str())),
@@ -772,7 +763,7 @@ pub(crate) fn dashboard_view(
today_hit_rate_value: today.observed_cached_token_rate,
yesterday_hit_rate_value: yesterday.observed_cached_token_rate,
total_hit_rate: display_rate(total_cached_token_rate),
total_cached_tokens: format_compact_number(requests.cached_tokens),
total_cached_tokens: format_compact_number(totals.cached_tokens),
average_first_token_latency_ms: display_duration(average_first_token_latency_ms),
},
},
@@ -783,14 +774,11 @@ pub(crate) fn dashboard_view(
usage_records: recent_requests.into_iter().map(usage_record_view).collect(),
pool_summary: DashboardPoolSummaryView {
total: provider_accounts.total,
active: provider_accounts.active,
rate_limited: provider_accounts.rate_limited,
expired: provider_accounts.expired,
invalid: provider_accounts.invalid,
normal: provider_accounts.normal,
quota_exhausted: provider_accounts.quota_exhausted,
refreshing: provider_accounts.refreshing,
rate_limited: provider_accounts.rate_limited,
disabled: provider_accounts.disabled,
banned: provider_accounts.banned,
error: provider_accounts.error,
},
capacity_info: DashboardCapacityInfoView {
max_concurrent_per_account: capacity.max_concurrent_per_account,
@@ -310,16 +310,6 @@ pub(crate) fn usage_range(
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
}
pub(crate) fn dashboard_range(
start: Option<&str>,
end: Option<&str>,
) -> Result<domain::TimeRange, WireValidationError> {
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
let start =
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
}
pub(crate) fn dashboard_today_range(
start: Option<&str>,
end: Option<&str>,
@@ -38,7 +38,8 @@ where
S: AdminSessionState + Send + Sync,
{
let kind = query.trend_kind().map_err(map_wire_error)?;
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
// 概览默认按中国时区当日统计,与单独趋势接口保持同一口径。
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let result = state
.admin_services()
@@ -84,6 +84,9 @@ pub struct UsageRecordView {
pub id: String,
pub request_id: String,
pub client_api_key_id: Option<String>,
pub routing_scope: String,
pub routing_group_refs: Vec<String>,
pub routing_group_names_snapshot: Vec<String>,
pub kind: String,
pub provider: Option<String>,
pub authentication_kind: Option<String>,
@@ -91,7 +94,7 @@ pub struct UsageRecordView {
pub account_email: Option<String>,
pub account_name: Option<String>,
pub route: String,
pub model: String,
pub model: Option<String>,
pub requested_model: Option<String>,
pub upstream_model: Option<String>,
pub service_tier: Option<String>,
@@ -180,7 +183,7 @@ pub struct UsageAttemptView {
pub attempt_index: u32,
pub trigger: String,
pub provider: String,
pub model: String,
pub model: Option<String>,
pub transport: String,
pub send_state: String,
pub outcome: String,
@@ -347,14 +350,11 @@ pub struct DashboardAccountRequestBucketView {
#[serde(rename_all = "camelCase")]
pub struct DashboardPoolSummaryView {
pub total: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub rate_limited: u64,
pub disabled: u64,
pub banned: u64,
pub error: u64,
}
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
@@ -1,7 +1,9 @@
//! Admin API 的展示层辅助:数值格式化。
//!
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
//! 当前 wire 仍保留 display 字段;统一由本模块提供,避免 accounts/observability
//! 各自维护格式化规则。
use gateway_core::accounting::Decimal;
/// 千分位格式化。
#[must_use]
@@ -37,3 +39,33 @@ pub fn format_compact_number(value: u64) -> String {
}
format_number(value)
}
/// 货币展示格式化。USD 常规金额保留两位,小于 1 美元时最多保留四位。
#[must_use]
pub fn format_decimal_currency(amount: &str, currency: &str) -> String {
if currency != "USD" {
return format!("{currency} {amount}");
}
const DECIMAL_SCALE: u128 = 10_000_000_000;
let scaled = amount.parse::<Decimal>().unwrap_or_default().scaled();
let precision = if scaled != 0 && scaled < DECIMAL_SCALE {
4_u32
} else {
2_u32
};
let rounding_unit = 10_u128.pow(10 - precision);
let rounded = (scaled + rounding_unit / 2) / rounding_unit;
let display_scale = 10_u128.pow(precision);
let whole = rounded / display_scale;
let fraction = rounded % display_scale;
let mut display = format!("{whole}.{fraction:0width$}", width = precision as usize);
while display.ends_with('0')
&& display
.split_once('.')
.is_some_and(|(_, fraction)| fraction.len() > 2)
{
display.pop();
}
format!("${display}")
}
+21 -3
View File
@@ -1,4 +1,4 @@
//! `/healthz` 对 Core、Store 与 Host worker 健康事实的聚合。
//! `/healthz` 对 Core、Store 与 Host 关键 worker 健康事实的聚合。
use std::sync::Arc;
use std::time::Duration;
@@ -6,8 +6,10 @@ use std::time::Duration;
use axum::{extract::State, http::StatusCode};
use futures::future::join_all;
use gateway_core::health::{
HealthProbe, HealthState, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState,
HealthProbe, HealthState, WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource,
WorkerRuntimeState,
};
use gateway_core::task::WorkerKind;
use crate::ApiState;
@@ -32,7 +34,13 @@ impl HealthStatus {
}
pub(crate) async fn healthy(&self) -> bool {
if !self.workers.snapshot().iter().all(worker_is_healthy) {
if !self
.workers
.snapshot()
.iter()
.filter(|worker| worker_affects_healthz(worker))
.all(worker_is_healthy)
{
return false;
}
let checks = self.probes.iter().map(|probe| probe.check());
@@ -54,6 +62,16 @@ pub(crate) async fn healthz(State(state): State<ApiState>) -> StatusCode {
}
}
fn worker_affects_healthz(worker: &WorkerHealthSnapshot) -> bool {
let WorkerHealthKey::Task(id) = &worker.key else {
return false;
};
!matches!(
id.kind(),
WorkerKind::OAuthRefresh | WorkerKind::QuotaCatalogHealth
)
}
fn worker_is_healthy(worker: &WorkerHealthSnapshot) -> bool {
match worker.state {
WorkerRuntimeState::Disabled | WorkerRuntimeState::Standby => true,
@@ -89,6 +89,7 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
| EngineError::ProviderMetadataMismatch
| EngineError::ContinuationPinMismatch
| EngineError::RequiredAccountMismatch
| EngineError::AccountOutsideClientScope
| EngineError::DownstreamCommitRequired
| EngineError::NoActiveAttempt
| EngineError::InvalidDeliveryState => {
@@ -0,0 +1,233 @@
//! Codex Images 非流式 HTTP adapter。
use std::net::SocketAddr;
use axum::{
body::{Body, Bytes},
extract::{Extension, State, connect_info::ConnectInfo},
http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
response::{IntoResponse, Response},
};
use gateway_core::engine::execution::StartedExecution;
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
use gateway_core::operation::{ImageRequest, ImageRequestKind, Operation, RawJsonPayload};
use serde_json::{Map, Value};
use crate::ApiState;
use crate::openai::{
auth::{authenticate_client, authentication_error_response},
error::{engine_error_response, gateway_error_response, protocol_error_response},
responses::{PendingExecution, ProtocolError, ProtocolErrorBody, request_client_context},
};
const OPENAI_PROTOCOL: &str = "openai";
const IMAGE_TURN_ID_CONTEXT_KEY: &str = "image_turn_id";
/// `POST /v1/images/generations`。
pub(crate) async fn image_generations(
State(state): State<ApiState>,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
handle_image_request(
state,
connect_info,
headers,
body,
ImageRequestKind::Generation,
"/v1/images/generations",
)
.await
}
/// `POST /v1/images/edits`。
pub(crate) async fn image_edits(
State(state): State<ApiState>,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
handle_image_request(
state,
connect_info,
headers,
body,
ImageRequestKind::Edit,
"/v1/images/edits",
)
.await
}
async fn handle_image_request(
state: ApiState,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
kind: ImageRequestKind,
endpoint: &'static str,
) -> Response {
let service = state.openai();
let client = match authenticate_client(service, &headers) {
Ok(client) => client,
Err(error) => return authentication_error_response(error),
};
let (client_ip, user_agent) = request_client_context(
&headers,
connect_info.map(|Extension(ConnectInfo(address))| address),
);
let operation = match image_operation(body, &headers, kind) {
Ok(operation) => operation,
Err(error) => return gateway_error_response(&error),
};
let started = match service
.start_image(client, operation, client_ip, user_agent, endpoint)
.await
{
Ok(started) => started,
Err(error) => return gateway_error_response(&error),
};
collect_image_response(started).await
}
fn image_operation(
body: Bytes,
headers: &HeaderMap,
kind: ImageRequestKind,
) -> Result<Operation, GatewayError> {
let mut context = Map::new();
if let Some(turn_id) = headers
.get("x-codex-image-turn-id")
.and_then(|value| value.to_str().ok())
{
context.insert(
IMAGE_TURN_ID_CONTEXT_KEY.to_owned(),
Value::String(turn_id.to_owned()),
);
}
let payload = RawJsonPayload::new(OPENAI_PROTOCOL, body)
.map_err(|_| {
GatewayError::new(
GatewayErrorKind::Internal,
"OpenAI protocol identifier is invalid",
)
})?
.with_context(context);
Ok(Operation::GenerateImage(ImageRequest::from_raw_json(
kind, payload,
)))
}
async fn collect_image_response(started: StartedExecution) -> Response {
let mut execution = PendingExecution::new(started.session);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
let events = match session.collect_uncommitted().await {
Ok(events) => events,
Err(error) => {
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
};
let body = match raw_json_body(events) {
Some(body) => body,
None => {
let response = invalid_upstream_response();
let response = execution.record_response_status(response).await;
execution.cancel_and_finalize().await;
return response;
}
};
let status = session
.response_status_code()
.and_then(|status| StatusCode::from_u16(status).ok())
.filter(StatusCode::is_success)
.unwrap_or(StatusCode::OK);
let response_headers = session.response_headers().to_vec();
let response = json_body_response(body, status, &response_headers);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
execution.cancel_and_finalize().await;
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
if !session.is_finalized() {
execution.cancel_and_finalize().await;
return invalid_upstream_response();
}
execution.disarm();
response
}
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
let mut body = None;
for event in events {
let (_, wire) = event.into_parts();
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
continue;
};
let Some(raw) = wire.into_raw_json_body() else {
continue;
};
if body.replace(raw).is_some() {
return None;
}
}
body
}
fn json_body_response(
body: Bytes,
status: StatusCode,
response_headers: &[ProviderResponseHeader],
) -> Response {
let mut response = Response::new(Body::from(body));
*response.status_mut() = status;
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
apply_response_headers(response, response_headers)
}
fn apply_response_headers(
mut response: Response,
response_headers: &[ProviderResponseHeader],
) -> Response {
let connection_options =
crate::openai::responses::response_connection_options(response_headers);
for header in response_headers {
if !crate::openai::responses::response_header_is_forwardable(
header.name(),
&connection_options,
) {
continue;
}
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
continue;
};
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
continue;
};
response.headers_mut().append(name, value);
}
response
}
fn invalid_upstream_response() -> Response {
protocol_error_response(
StatusCode::BAD_GATEWAY,
ProtocolErrorBody {
error: ProtocolError {
kind: "server_error",
code: "invalid_upstream_response",
message: "The gateway could not forward the upstream image response.".to_owned(),
param: None,
},
},
)
.into_response()
}
@@ -0,0 +1,5 @@
//! OpenAI/Codex Images JSON 透明 adapter。
mod http;
pub(crate) use http::{image_edits, image_generations};
@@ -2,6 +2,7 @@
pub mod auth;
pub mod error;
pub mod images;
pub mod models;
pub mod responses;
pub mod router;
@@ -123,7 +123,7 @@ async fn handle_responses(
}
/// 从 socket 与标准转发头提取旧 Usage 页面使用的诊断事实。
pub(super) fn request_client_context(
pub(in crate::openai) fn request_client_context(
headers: &HeaderMap,
peer_address: Option<SocketAddr>,
) -> (Option<IpAddr>, Option<String>) {
@@ -367,22 +367,24 @@ fn internal_gateway_response(message: &'static str) -> Response {
gateway_error_response(&GatewayError::new(GatewayErrorKind::Internal, message))
}
pub(super) struct PendingExecution {
pub(in crate::openai) struct PendingExecution {
session: Option<Box<dyn ExecutionSession>>,
}
impl PendingExecution {
pub(super) fn new(session: Box<dyn ExecutionSession>) -> Self {
pub(in crate::openai) fn new(session: Box<dyn ExecutionSession>) -> Self {
Self {
session: Some(session),
}
}
pub(super) fn session_mut(&mut self) -> Option<&mut (dyn ExecutionSession + 'static)> {
pub(in crate::openai) fn session_mut(
&mut self,
) -> Option<&mut (dyn ExecutionSession + 'static)> {
self.session.as_deref_mut()
}
async fn cancel_and_finalize(&mut self) {
pub(in crate::openai) async fn cancel_and_finalize(&mut self) {
if let Some(session) = self.session.as_mut() {
session.cancel();
if !session.is_finalized() {
@@ -391,7 +393,10 @@ impl PendingExecution {
}
}
async fn record_response_status(&mut self, response: Response) -> Response {
pub(in crate::openai) async fn record_response_status(
&mut self,
response: Response,
) -> Response {
if let Some(session) = self.session.as_mut() {
let _ = session
.record_client_status(response.status().as_u16())
@@ -400,7 +405,7 @@ impl PendingExecution {
response
}
pub(super) fn disarm(&mut self) {
pub(in crate::openai) fn disarm(&mut self) {
self.session = None;
}
@@ -6,7 +6,8 @@ mod request;
mod response;
mod websocket;
pub use error::{ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
pub(super) use http::{PendingExecution, request_client_context};
pub use http::{collect_execution_response, stream_execution_response};
pub(crate) use http::{responses, review_responses};
pub use request::{
@@ -73,7 +74,6 @@ pub(super) fn response_header_is_forwardable(name: &str, connection_options: &[S
| "x-openai-organization"
| "x-openai-project"
| "x-codex-installation-id"
| "x-codex-turn-state"
| "x-codex-turn-metadata"
)
}
@@ -165,7 +165,6 @@ impl fmt::Debug for ContinuationIntent {
#[derive(Clone, PartialEq, Eq)]
pub struct ResponsesRequestMetadata {
requested_model: String,
public_model: String,
stream: bool,
store: bool,
continuation: ContinuationIntent,
@@ -180,12 +179,6 @@ impl ResponsesRequestMetadata {
&self.requested_model
}
/// 返回协议层公开模型候选。
#[must_use]
pub fn public_model(&self) -> &str {
&self.public_model
}
/// 返回客户端是否请求 SSE。
#[must_use]
pub const fn stream(&self) -> bool {
@@ -222,7 +215,6 @@ impl fmt::Debug for ResponsesRequestMetadata {
formatter
.debug_struct("ResponsesRequestMetadata")
.field("requested_model", &self.requested_model)
.field("public_model", &self.public_model)
.field("stream", &self.stream)
.field("store", &self.store)
.field("continuation", &self.continuation)
@@ -366,8 +358,7 @@ pub(super) fn decode_request_object(
Ok(DecodedResponsesRequest {
operation,
metadata: ResponsesRequestMetadata {
requested_model: model.clone(),
public_model: model,
requested_model: model,
stream,
store,
continuation,
@@ -7,6 +7,7 @@ use axum::{
};
use super::{
images::{image_edits, image_generations},
models::{model_catalog, model_detail, model_info, models},
responses::{responses, responses_websocket, review_responses},
};
@@ -16,6 +17,8 @@ use crate::ApiState;
/// 构造 OpenAI 客户端协议路由。
pub(crate) fn router() -> Router<ApiState> {
Router::new()
.route("/v1/images/generations", post(image_generations))
.route("/v1/images/edits", post(image_edits))
.route("/v1/responses", get(responses_websocket).post(responses))
.route("/v1/responses/review", post(review_responses))
.route("/v1/models", get(models))
@@ -23,7 +26,7 @@ pub(crate) fn router() -> Router<ApiState> {
.route("/v1/models/{model_id}/info", get(model_info))
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
.route("/v1/models/{model_id}", get(model_detail))
// Responses 正文属于 OpenAI/Codex 协议;代理不能用私有大小上限提前拒绝
// OpenAI 数据面正文属于客户端/上游协议;代理不能用私有大小上限提前拒绝
// 上游本可接受的未来 payload。
.layer(DefaultBodyLimit::disable())
}
@@ -1,11 +1,12 @@
//! OpenAI wire adapter 到 Core 执行用例的唯一映射。
use std::net::IpAddr;
use std::sync::Arc;
use gateway_core::engine::continuation::PreviousResponseId;
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionRequestMetadata,
ExecutionService, StartExecution, StartedExecution,
ExecutionService, StartExecution, StartProviderExecution, StartedExecution,
};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLifecycle};
@@ -74,13 +75,13 @@ impl OpenAiService {
endpoint: &'static str,
) -> Result<StartedExecution, GatewayError> {
let (operation, metadata) = request.into_parts();
let public_model = PublicModelId::from_client_wire(metadata.public_model().to_owned())
let public_model = PublicModelId::from_client_wire(metadata.requested_model().to_owned())
.map_err(|_| {
GatewayError::new(
GatewayErrorKind::ModelNotFound,
"requested model was not found",
)
})?;
GatewayError::new(
GatewayErrorKind::ModelNotFound,
"requested model was not found",
)
})?;
let previous_response_id = match metadata.continuation() {
ContinuationIntent::None => None,
ContinuationIntent::PreviousResponseId(value) => {
@@ -105,6 +106,38 @@ impl OpenAiService {
.await
}
pub(crate) async fn start_image(
&self,
client: AuthenticatedClient,
operation: gateway_core::operation::Operation,
client_ip: Option<IpAddr>,
user_agent: Option<String>,
endpoint: &'static str,
) -> Result<StartedExecution, GatewayError> {
let provider = gateway_core::routing::ProviderKind::new("openai").map_err(|_| {
GatewayError::new(
GatewayErrorKind::Internal,
"OpenAI provider identifier is invalid",
)
})?;
self.execution
.start_provider_endpoint(StartProviderExecution {
client,
provider,
operation,
metadata: ExecutionRequestMetadata {
protocol: "openai".to_owned(),
endpoint: endpoint.to_owned(),
transport: ClientTransport::HttpJson,
stream: false,
client_ip,
user_agent,
previous_response_id: None,
},
})
.await
}
pub(crate) fn try_register_connection(
&self,
) -> Result<Box<dyn ConnectionGuard>, ConnectionDraining> {
@@ -0,0 +1,271 @@
use axum::{
Router,
body::{Body, to_bytes},
http::{Method, Request, StatusCode, header},
};
use gateway_api::admin::account_groups;
use serde_json::{Value, json};
use tower::ServiceExt as _;
use super::{AdminTestFixture, AdminTestState, PRIMARY_GROUP_ID};
const SESSION_COOKIE: &str = "cpr_admin_session=valid-session";
#[tokio::test]
async fn list_route_should_keep_camel_case_group_and_page_wire() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::GET,
"/api/admin/account-groups?page=1&pageSize=1&search=alpha&enabled=true",
None,
true,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let value = response_json(response).await;
assert_eq!(value["data"]["page"]["page"], 1);
assert_eq!(value["data"]["page"]["pageSize"], 1);
assert_eq!(value["data"]["page"]["total"], 1);
assert_eq!(value["data"]["page"]["totalPages"], 1);
assert_eq!(value["data"]["configRevision"], 7);
assert_eq!(value["data"]["items"][0]["id"], PRIMARY_GROUP_ID);
assert_eq!(value["data"]["items"][0]["memberCount"], 2);
assert_eq!(value["data"]["items"][0]["providerCounts"]["openai"], 1);
assert_eq!(value["data"]["items"][0]["clientKeyCount"], 2);
assert_eq!(value["data"]["items"][0]["accountSummary"]["available"], 1);
assert_eq!(value["data"]["items"][0]["accountSummary"]["limited"], 1);
assert_eq!(value["data"]["items"][0]["accountSummary"]["total"], 2);
assert_eq!(value["data"]["items"][0]["capacity"]["usedSlots"], 0);
assert_eq!(value["data"]["items"][0]["capacity"]["totalSlots"], 1);
assert_eq!(value["data"]["items"][0]["usage"]["todayUsd"], "1.25");
assert_eq!(value["data"]["items"][0]["usage"]["totalUsd"], "5.5");
assert!(value["data"]["items"][0].get("member_count").is_none());
}
#[tokio::test]
async fn create_route_should_create_an_empty_group() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/create",
Some(json!({
"name": "Gamma routing",
"description": "New traffic pool",
"color": "#a855f780"
})),
true,
)
.await;
assert_eq!(response.status(), StatusCode::CREATED);
let value = response_json(response).await;
assert!(
value["data"]["id"]
.as_str()
.is_some_and(|id| id.starts_with("grp_") && id.len() == 36)
);
assert_eq!(value["data"]["record"]["name"], "Gamma routing");
assert_eq!(value["data"]["record"]["memberCount"], 0);
assert_eq!(value["data"]["record"]["color"], "#A855F780");
assert_eq!(value["data"]["configRevision"], 8);
}
#[tokio::test]
async fn update_route_should_replace_group_fields() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/update",
Some(json!({
"id": PRIMARY_GROUP_ID,
"name": "Renamed routing",
"description": null,
"color": "#F43F5ECC"
})),
true,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let value = response_json(response).await;
assert_eq!(value["data"]["record"]["name"], "Renamed routing");
assert!(value["data"]["record"]["description"].is_null());
assert_eq!(value["data"]["record"]["color"], "#F43F5ECC");
}
#[tokio::test]
async fn enable_and_disable_routes_should_set_explicit_group_state() {
let fixture = authenticated_fixture().await;
let disabled = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/disable",
Some(json!({ "id": PRIMARY_GROUP_ID })),
true,
)
.await;
let enabled = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/enable",
Some(json!({ "id": PRIMARY_GROUP_ID })),
true,
)
.await;
assert_eq!(disabled.status(), StatusCode::OK);
assert!(
!response_json(disabled).await["data"]["record"]["enabled"]
.as_bool()
.expect("disabled flag")
);
assert_eq!(enabled.status(), StatusCode::OK);
assert!(
response_json(enabled).await["data"]["record"]["enabled"]
.as_bool()
.expect("enabled flag")
);
}
#[tokio::test]
async fn delete_route_should_return_deleted_id_without_a_live_record() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/delete",
Some(json!({ "id": PRIMARY_GROUP_ID })),
true,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let value = response_json(response).await;
assert_eq!(value["data"]["id"], PRIMARY_GROUP_ID);
assert!(value["data"]["record"].is_null());
}
#[tokio::test]
async fn list_route_should_reject_unknown_and_invalid_pagination_fields() {
let fixture = authenticated_fixture().await;
for uri in [
"/api/admin/account-groups?other=true",
"/api/admin/account-groups?page=0",
"/api/admin/account-groups?pageSize=0",
"/api/admin/account-groups?pageSize=201",
] {
let response = request(router(&fixture), Method::GET, uri, None, true).await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}");
}
}
#[tokio::test]
async fn mutation_route_should_reject_unknown_json_fields() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/create",
Some(json!({
"name": "Unknown field",
"description": null,
"color": "#2563EBFF",
"providerKind": "openai"
})),
true,
)
.await;
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
}
#[tokio::test]
async fn create_route_should_reject_six_digit_color_without_alpha() {
let fixture = authenticated_fixture().await;
let response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/create",
Some(json!({
"name": "Opaque legacy color",
"description": null,
"color": "#2563EB"
})),
true,
)
.await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn account_group_routes_should_require_admin_authentication() {
let fixture = AdminTestFixture::new().await;
let get_response = request(
router(&fixture),
Method::GET,
"/api/admin/account-groups",
None,
false,
)
.await;
let post_response = request(
router(&fixture),
Method::POST,
"/api/admin/account-groups/create",
Some(json!({ "name": "Unauthorized", "description": null, "color": "#2563EBFF" })),
false,
)
.await;
assert_eq!(get_response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(post_response.status(), StatusCode::UNAUTHORIZED);
}
async fn authenticated_fixture() -> AdminTestFixture {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
fixture
}
fn router(fixture: &AdminTestFixture) -> Router {
account_groups::router::<AdminTestState>().with_state(fixture.state())
}
async fn request(
router: Router,
method: Method,
uri: &str,
body: Option<Value>,
authenticated: bool,
) -> axum::response::Response {
let mut builder = Request::builder()
.method(method)
.uri(uri)
.header("x-request-id", "req_account_groups");
if authenticated {
builder = builder.header(header::COOKIE, SESSION_COOKIE);
}
let body = match body {
Some(value) => {
builder = builder.header(header::CONTENT_TYPE, "application/json");
Body::from(serde_json::to_vec(&value).expect("serialize request body"))
}
None => Body::empty(),
};
router
.oneshot(builder.body(body).expect("account group request"))
.await
.expect("account group response")
}
async fn response_json(response: axum::response::Response) -> Value {
let body = to_bytes(response.into_body(), 64 * 1024)
.await
.expect("account group response body");
serde_json::from_slice(&body).expect("account group response JSON")
}
@@ -70,8 +70,138 @@ mod query {
}
}
mod batch_update {
use gateway_api::admin::accounts::{BatchUpdateAccountsRequest, UpdateAccountRequest};
use serde_json::json;
#[test]
fn batch_update_should_accept_complete_atomic_payload() {
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
"accountIds": ["acct_openai", "acct_xai"],
"enabled": false,
"concurrencyLimit": null,
"weight": 1,
"groupIds": ["grp_00000000000000000000000000000001"]
}))
.expect("deserialize batch update");
request.validate().expect("validate batch update");
}
#[test]
fn batch_update_should_reject_duplicate_or_invalid_account_ids() {
for account_ids in [
json!(["acct_same", "acct_same"]),
json!(["not-an-account"]),
json!([]),
] {
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
"accountIds": account_ids,
"enabled": true,
"concurrencyLimit": 8,
"weight": 100,
"groupIds": []
}))
.expect("deserialize invalid batch update");
assert_eq!(
request.validate().expect_err("reject account IDs").field(),
"accountIds"
);
}
}
#[test]
fn batch_update_should_require_enabled_and_reject_unknown_fields() {
assert!(
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
"accountIds": ["acct_test"],
"concurrencyLimit": null,
"weight": 1,
"groupIds": []
}))
.is_err()
);
assert!(
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
"accountIds": ["acct_test"],
"enabled": true,
"concurrencyLimit": null,
"weight": 1,
"groupIds": [],
"legacy": true
}))
.is_err()
);
}
#[test]
fn batch_update_should_reject_invalid_scheduling_bounds_and_missing_fields() {
for (concurrency_limit, weight, field) in [
(json!(0), json!(1), "concurrencyLimit"),
(json!(4294967296_u64), json!(1), "concurrencyLimit"),
(json!(null), json!(0), "weight"),
(json!(null), json!(101), "weight"),
] {
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
"accountIds": ["acct_test"],
"enabled": true,
"concurrencyLimit": concurrency_limit,
"weight": weight,
"groupIds": []
}))
.expect("deserialize invalid scheduling");
assert_eq!(
request.validate().expect_err("reject scheduling").field(),
field
);
}
assert!(
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
"accountIds": ["acct_test"],
"enabled": true,
"weight": 1,
"groupIds": []
}))
.is_err()
);
assert!(
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
"accountIds": ["acct_test"],
"enabled": true,
"concurrencyLimit": null,
"groupIds": []
}))
.is_err()
);
}
#[test]
fn single_update_should_require_the_same_complete_scheduling_contract() {
let request: UpdateAccountRequest = serde_json::from_value(json!({
"accountId": "acct_test",
"enabled": true,
"concurrencyLimit": 4294967295_u64,
"weight": 100,
"groupIds": []
}))
.expect("deserialize single update");
request.validate().expect("validate single update");
assert!(
serde_json::from_value::<UpdateAccountRequest>(json!({
"accountId": "acct_test",
"enabled": true,
"weight": 1,
"groupIds": []
}))
.is_err()
);
}
}
mod response {
use gateway_api::admin::accounts::AccountUsageView;
use gateway_api::admin::presenter::format_decimal_currency;
#[test]
fn account_usage_view_should_keep_unobserved_numbers_null() {
@@ -112,12 +242,22 @@ mod response {
assert!(value["totalTokens"].is_null());
assert_eq!(value["createdTokensDisplay"], "-");
}
#[test]
fn account_usage_currency_should_limit_usd_to_four_fraction_digits() {
assert_eq!(format_decimal_currency("0.1204956", "USD"), "$0.1205");
assert_eq!(format_decimal_currency("0.99996", "USD"), "$1.00");
assert_eq!(format_decimal_currency("0.0300", "USD"), "$0.03");
assert_eq!(format_decimal_currency("12.3456", "USD"), "$12.35");
assert_eq!(format_decimal_currency("0.1204956", "CNY"), "CNY 0.1204956");
}
}
mod actions {
use base64::Engine as _;
use gateway_admin::model::{
Revision,
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
accounts::AccountConnectionTestEvent as DomainConnectionTestEvent,
provider_credentials::{
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
},
@@ -126,7 +266,8 @@ mod actions {
AccountActionRequest, AccountConnectionTestEvent, AccountDeletionData,
AccountDeletionRequest, AccountExportData, AccountExportQuery, AccountIdQuery,
AccountImportData, AccountImportRequest, AccountMutationData, AccountRefreshRequest,
AccountTestQuery, RotateAccountRequest, StartAccountAuthorizationRequest,
AccountResetCreditConsumeRequest, AccountTestQuery, CompleteAccountAuthorizationRequest,
RotateAccountRequest, StartAccountAuthorizationRequest,
};
use gateway_core::engine::credential::ProviderAccountId;
use serde_json::json;
@@ -184,6 +325,44 @@ mod actions {
);
}
#[test]
fn reset_credit_consume_should_require_canonical_v4_idempotency_key() {
let valid: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
"accountId": "acct_1",
"creditId": "credit_1",
"redeemRequestId": "8fbf302d-11df-4bd5-82e4-08e4b3df7874"
}))
.expect("decode reset-credit consume");
valid.validate().expect("validate reset-credit consume");
for (redeem_request_id, field) in [
("019c0000-0000-7000-8000-000000000000", "redeemRequestId"),
("8FBF302D-11DF-4BD5-82E4-08E4B3DF7874", "redeemRequestId"),
("invalid", "redeemRequestId"),
] {
let request: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
"accountId": "acct_1",
"redeemRequestId": redeem_request_id
}))
.expect("decode invalid reset-credit consume");
assert_eq!(request.validate().expect_err("reject UUID").field(), field);
}
let invalid_credit: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
"accountId": "acct_1",
"creditId": " ",
"redeemRequestId": "8fbf302d-11df-4bd5-82e4-08e4b3df7874"
}))
.expect("decode invalid credit");
assert_eq!(
invalid_credit
.validate()
.expect_err("reject credit")
.field(),
"creditId"
);
}
#[test]
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
@@ -207,7 +386,8 @@ mod actions {
"provider": "openai",
"accountId": "acct_1",
"accessToken": "header.payload.signature",
"refreshToken": "refresh-token"
"refreshToken": "refresh-token",
"idToken": "id-header.id-payload.id-signature"
}))
.expect("decode rotation");
assert!(rotation.validate().is_ok());
@@ -269,6 +449,35 @@ mod actions {
}))
.is_err()
);
assert!(
serde_json::from_value::<AccountImportRequest>(json!({
"provider": "openai",
"data": {},
"groupIds": []
}))
.is_err()
);
}
#[test]
fn oauth_complete_should_not_accept_group_assignment() {
let flow_id = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7_u8; 32]);
let request: CompleteAccountAuthorizationRequest = serde_json::from_value(json!({
"provider": "openai",
"flowId": flow_id,
"callbackUrl": "http://localhost/callback?code=ok"
}))
.expect("decode OAuth completion");
assert!(request.validate().is_ok());
assert!(
serde_json::from_value::<CompleteAccountAuthorizationRequest>(json!({
"provider": "xai",
"flowId": "flow_test",
"callbackUrl": "http://localhost/callback?code=ok",
"groupIds": []
}))
.is_err()
);
}
#[test]
@@ -305,7 +514,7 @@ mod actions {
#[test]
fn account_import_response_should_emit_account_ids() {
let response = AccountImportData::from(CredentialImportResult {
let response = AccountImportData::from_result(CredentialImportResult {
config_revision: Revision::new(8).expect("revision"),
credential_ids: vec![ProviderAccountId::new("acct_imported").expect("account ID")],
});
@@ -343,9 +552,7 @@ mod actions {
DomainConnectionTestEvent::Content {
text: "OK".to_owned(),
},
DomainConnectionTestEvent::Completed {
account_status: AccountStatus::Normal,
},
DomainConnectionTestEvent::Completed {},
DomainConnectionTestEvent::Failed {
message: "upstream unavailable".to_owned(),
provider_error_code: Some("usage_exhausted".to_owned()),
@@ -353,7 +560,6 @@ mod actions {
upstream_status: Some(429),
upstream_content_type: Some("application/json".to_owned()),
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
account_status: AccountStatus::QuotaExhausted,
},
]
.map(|event| AccountConnectionTestEvent::from(event).data);
@@ -378,7 +584,7 @@ mod actions {
}
}),
json!({ "type": "content", "text": "OK" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
json!({ "type": "test_complete", "success": true }),
json!({
"type": "error",
"error": "upstream unavailable",
@@ -386,8 +592,7 @@ mod actions {
"providerErrorType": "invalid_request_error",
"upstreamStatus": 429,
"upstreamContentType": "application/json",
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
"accountStatus": "quota_exhausted"
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#
}),
]
);
@@ -105,7 +105,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
let valid = serde_json::from_value::<CreateClientKeyRequest>(json!({
"name": "terminal key",
"label": "production",
"providerKind": "openai",
"groupIds": [],
"maxConcurrency": 2,
"requestsPerMinute": 60
}))
@@ -118,7 +118,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
(
json!({
"name": " ",
"providerKind": "openai",
"groupIds": [],
"maxConcurrency": 0,
"requestsPerMinute": 0
}),
@@ -127,7 +127,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
(
json!({
"name": "key",
"providerKind": "openai",
"groupIds": [],
"maxConcurrency": u64::MAX,
"requestsPerMinute": 0
}),
@@ -150,7 +150,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
"id": "key_1",
"expectedConfigRevision": 1,
"name": "key",
"providerKind": "openai",
"groupIds": [],
"maxConcurrency": 0,
"requestsPerMinute": 0,
"tokensPerMinute": 0
@@ -161,7 +161,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
serde_json::from_value::<CreateClientKeyRequest>(json!({
"expectedConfigRevision": 7,
"name": "terminal key",
"providerKind": "openai",
"groupIds": [],
"maxConcurrency": 2,
"requestsPerMinute": 60
}))
@@ -210,7 +210,10 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
id: gateway_core::policy::ClientApiKeyId::new("key_visible").expect("Client Key ID"),
name: "visible".to_owned(),
label: None,
provider_kind: gateway_core::routing::ProviderKind::new("openai").expect("Provider kind"),
groups: Vec::new(),
provider_kinds: vec![
gateway_core::routing::ProviderKind::new("openai").expect("Provider kind"),
],
prefix: "sk_visible12".to_owned(),
enabled: true,
limits: gateway_core::policy::RateLimits {
@@ -226,7 +229,11 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
assert!(list.get("configRevision").is_none());
assert_eq!(list["total"], 1);
assert_eq!(list["items"][0]["id"], "key_visible");
assert_eq!(list["items"][0]["providerKind"], "openai");
assert_eq!(
list["items"][0]["providerKinds"],
serde_json::json!(["openai"])
);
assert_eq!(list["items"][0]["routingScope"], "all");
assert_eq!(list["items"][0]["maxConcurrency"], 2);
assert_eq!(list["items"][0]["requestsPerMinute"], 60);
assert!(list["items"][0].get("tokensPerMinute").is_none());
+282 -14
View File
@@ -7,15 +7,22 @@ use std::{
};
use async_trait::async_trait;
use chrono::{Duration, Utc};
use chrono::{DateTime, Duration, Utc};
use futures::future::BoxFuture;
use gateway_admin::{
AdminConfig, AdminServices, InitialAdminPassword,
model::{
MutationContext, Revision,
account_groups::{
AccountGroupAccountSummary, AccountGroupCapacity, AccountGroupColor,
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, AccountGroupRecord,
AccountGroupUsage, DeleteAccountGroup, NewAccountGroup, SetAccountGroupEnabled,
UpdateAccountGroup,
},
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
AccountListQuery, AccountPage, AccountPageItem, AccountUpdateResult, AccountUsage,
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
},
auth::{AdminAuditEvent, AdminSession},
client_keys::{
@@ -23,9 +30,9 @@ use gateway_admin::{
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
},
observability::{
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
UsageOverview, UsagePage, UsageQuery, UsageRecord,
DashboardObservation, DecimalAmount, DiagnosticDimension, DiagnosticObservation,
OpsError, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail,
UsageFilter, UsageOverview, UsagePage, UsageQuery, UsageRecord,
},
provider_credentials::{
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
@@ -45,8 +52,8 @@ use gateway_admin::{
ports::{
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
store::{
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
AccountGroupStore, AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts,
AdminStoreResult, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
},
system::{
SystemOperationError, SystemOperationErrorKind, SystemOperations,
@@ -66,6 +73,7 @@ use gateway_core::{
},
};
mod account_groups;
mod accounts;
mod auth;
mod client_keys;
@@ -83,6 +91,7 @@ pub(super) struct AdminTestFixture {
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
}
impl AdminTestFixture {
@@ -95,20 +104,24 @@ impl AdminTestFixture {
let auth = Arc::new(MemoryAuthStore::new(api_key.clone()));
let settings = Arc::new(MemorySettingsStore::new(api_key));
let client_keys = Arc::new(MemoryClientKeyStore);
let account_groups = Arc::new(MemoryAccountGroupStore::new());
let usage_records = Arc::new(Mutex::new(Vec::new()));
let usage_detail = Arc::new(Mutex::new(None));
let diagnostics = Arc::new(Mutex::new(Vec::new()));
let ops_errors = Arc::new(Mutex::new(Vec::new()));
let dashboard_observation = Arc::new(Mutex::new(None));
let dashboard_summary_range = Arc::new(Mutex::new(None));
let unused = Arc::new(UnusedStore {
usage_records: Arc::clone(&usage_records),
usage_detail: Arc::clone(&usage_detail),
diagnostics: Arc::clone(&diagnostics),
ops_errors: Arc::clone(&ops_errors),
dashboard_observation: Arc::clone(&dashboard_observation),
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
});
let stores = AdminStorePorts::new(
unused.clone(),
account_groups.clone(),
auth.clone(),
client_keys.clone(),
unused,
@@ -142,6 +155,7 @@ impl AdminTestFixture {
diagnostics,
ops_errors,
dashboard_observation,
dashboard_summary_range,
}
}
@@ -341,6 +355,204 @@ impl SettingsStore for MemorySettingsStore {
pub(super) struct MemoryClientKeyStore;
pub(super) const PRIMARY_GROUP_ID: &str = "grp_11111111111111111111111111111111";
pub(super) const SECONDARY_GROUP_ID: &str = "grp_22222222222222222222222222222222";
struct MemoryAccountGroupState {
revision: Revision,
groups: BTreeMap<gateway_core::routing::AccountGroupId, AccountGroupRecord>,
}
pub(super) struct MemoryAccountGroupStore {
state: Mutex<MemoryAccountGroupState>,
}
impl MemoryAccountGroupStore {
fn new() -> Self {
let now = Utc::now();
let primary_id = group_id(PRIMARY_GROUP_ID);
let secondary_id = group_id(SECONDARY_GROUP_ID);
let groups = BTreeMap::from([
(
primary_id.clone(),
AccountGroupRecord {
id: primary_id,
name: "Alpha routing".to_owned(),
description: Some("Primary traffic".to_owned()),
color: group_color("#2563ebff"),
enabled: true,
member_count: 2,
provider_counts: BTreeMap::from([
("openai".to_owned(), 1),
("xai".to_owned(), 1),
]),
client_key_count: 2,
account_summary: account_summary(1, 1, 2),
capacity: capacity(Some(0), 1),
usage: usage("1.25", "5.5"),
created_at: now,
updated_at: now,
},
),
(
secondary_id.clone(),
AccountGroupRecord {
id: secondary_id,
name: "Beta routing".to_owned(),
description: None,
color: group_color("#64748B80"),
enabled: false,
member_count: 0,
provider_counts: BTreeMap::new(),
client_key_count: 0,
account_summary: account_summary(0, 0, 0),
capacity: capacity(Some(0), 0),
usage: usage("0", "0"),
created_at: now,
updated_at: now,
},
),
]);
Self {
state: Mutex::new(MemoryAccountGroupState {
revision: Revision::new(7).expect("revision"),
groups,
}),
}
}
}
#[async_trait]
impl AccountGroupStore for MemoryAccountGroupStore {
async fn list_account_groups(
&self,
query: AccountGroupListQuery,
) -> AdminStoreResult<AccountGroupPage> {
let state = self.state.lock().expect("account groups");
let search = query.search.as_ref().map(|value| value.to_lowercase());
let mut matching = state
.groups
.values()
.filter(|record| {
query
.enabled
.is_none_or(|enabled| record.enabled == enabled)
})
.filter(|record| {
search.as_ref().is_none_or(|search| {
record.name.to_lowercase().contains(search)
|| record
.description
.as_ref()
.is_some_and(|description| description.to_lowercase().contains(search))
})
})
.cloned()
.collect::<Vec<_>>();
matching.sort_by(|left, right| left.name.cmp(&right.name));
let total = matching.len() as u64;
let page_size = usize::from(query.page_size.get());
let offset = usize::try_from(query.page.saturating_sub(1))
.unwrap_or(usize::MAX)
.saturating_mul(page_size);
let items = matching.into_iter().skip(offset).take(page_size).collect();
Ok(AccountGroupPage {
config_revision: state.revision,
items,
total,
page: query.page,
page_size: query.page_size.get(),
})
}
async fn create_account_group(
&self,
command: NewAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
let mut state = self.state.lock().expect("account groups");
let now = Utc::now();
let record = AccountGroupRecord {
id: command.id.clone(),
name: command.name,
description: command.description,
color: command.color,
enabled: true,
member_count: 0,
provider_counts: BTreeMap::new(),
client_key_count: 0,
account_summary: account_summary(0, 0, 0),
capacity: capacity(Some(0), 0),
usage: usage("0", "0"),
created_at: now,
updated_at: now,
};
state.groups.insert(command.id.clone(), record);
mutation(&mut state, command.id, true)
}
async fn update_account_group(
&self,
command: UpdateAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
let mut state = self.state.lock().expect("account groups");
let record = state
.groups
.get_mut(&command.id)
.ok_or_else(|| not_found("account group"))?;
record.name = command.name;
record.description = command.description;
record.color = command.color;
record.updated_at = Utc::now();
mutation(&mut state, command.id, true)
}
async fn set_account_group_enabled(
&self,
command: SetAccountGroupEnabled,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
let mut state = self.state.lock().expect("account groups");
let record = state
.groups
.get_mut(&command.id)
.ok_or_else(|| not_found("account group"))?;
record.enabled = command.enabled;
record.updated_at = Utc::now();
mutation(&mut state, command.id, true)
}
async fn delete_account_group(
&self,
command: DeleteAccountGroup,
_: &MutationContext,
) -> AdminStoreResult<AccountGroupMutation> {
let mut state = self.state.lock().expect("account groups");
if state.groups.remove(&command.id).is_none() {
return Err(not_found("account group"));
}
mutation(&mut state, command.id, false)
}
}
fn group_id(value: &str) -> gateway_core::routing::AccountGroupId {
gateway_core::routing::AccountGroupId::new(value).expect("account group ID")
}
fn mutation(
state: &mut MemoryAccountGroupState,
id: gateway_core::routing::AccountGroupId,
include_record: bool,
) -> AdminStoreResult<AccountGroupMutation> {
state.revision = next_revision(state.revision);
Ok(AccountGroupMutation {
config_revision: state.revision,
record: include_record.then(|| state.groups.get(&id).expect("group record").clone()),
id,
})
}
#[async_trait]
impl ClientKeyStore for MemoryClientKeyStore {
async fn list_client_keys(&self, _: ClientKeyListQuery) -> AdminStoreResult<ClientKeyPage> {
@@ -362,7 +574,8 @@ impl ClientKeyStore for MemoryClientKeyStore {
id: id.clone(),
name: "revealed".to_owned(),
label: None,
provider_kind: ProviderKind::new("openai").expect("provider kind"),
groups: Vec::new(),
provider_kinds: vec![ProviderKind::new("openai").expect("provider kind")],
prefix: "sk_aaaaaaaaa".to_owned(),
enabled: true,
limits: RateLimits::unlimited(),
@@ -413,6 +626,7 @@ struct UnusedStore {
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
ops_errors: Arc<Mutex<Vec<OpsError>>>,
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
}
#[async_trait]
@@ -421,7 +635,7 @@ impl AccountStore for UnusedStore {
Err(unavailable("account list"))
}
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountPageItem>> {
Err(unavailable("account"))
}
@@ -496,14 +710,30 @@ impl AccountStore for UnusedStore {
Err(unavailable("credential refresh"))
}
async fn set_account_enabled(
async fn update_account(
&self,
_: SetAccountEnabled,
_: UpdateAccount,
_: &MutationContext,
) -> AdminStoreResult<Revision> {
) -> AdminStoreResult<AccountUpdateResult> {
Err(unavailable("account enabled"))
}
async fn recover_account(
&self,
_: &ProviderAccountId,
_: &MutationContext,
) -> AdminStoreResult<AccountUpdateResult> {
Err(unavailable("account recovery"))
}
async fn batch_update_accounts(
&self,
_: BatchUpdateAccounts,
_: &MutationContext,
) -> AdminStoreResult<AccountsUpdateResult> {
Err(unavailable("account batch update"))
}
async fn delete_accounts(
&self,
_: DeleteAccounts,
@@ -523,7 +753,15 @@ impl AccountStore for UnusedStore {
#[async_trait]
impl ObservabilityStore for UnusedStore {
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
async fn dashboard_summary(
&self,
range: TimeRange,
_: DateTime<Utc>,
) -> AdminStoreResult<DashboardObservation> {
*self
.dashboard_summary_range
.lock()
.expect("dashboard summary range") = Some(range);
self.dashboard_observation
.lock()
.expect("dashboard observation")
@@ -779,6 +1017,32 @@ fn next_revision(revision: Revision) -> Revision {
Revision::new(revision.get().saturating_add(1)).expect("next revision")
}
fn account_summary(available: u64, limited: u64, total: u64) -> AccountGroupAccountSummary {
AccountGroupAccountSummary {
available,
limited,
total,
}
}
fn group_color(value: &str) -> AccountGroupColor {
AccountGroupColor::parse(value).expect("group color")
}
fn capacity(used_slots: Option<u64>, total_slots: u64) -> AccountGroupCapacity {
AccountGroupCapacity {
used_slots,
total_slots,
}
}
fn usage(today_usd: &str, total_usd: &str) -> AccountGroupUsage {
AccountGroupUsage {
today_usd: today_usd.parse::<DecimalAmount>().expect("today cost"),
total_usd: total_usd.parse::<DecimalAmount>().expect("total cost"),
}
}
fn unavailable(resource: &'static str) -> AdminStoreError {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
@@ -787,6 +1051,10 @@ fn unavailable(resource: &'static str) -> AdminStoreError {
)
}
fn not_found(resource: &'static str) -> AdminStoreError {
AdminStoreError::new(AdminStoreErrorKind::NotFound, resource, "not found")
}
fn unsupported_provider() -> ProviderAdminError {
ProviderAdminError::new(ProviderAdminErrorKind::Unsupported)
}
@@ -219,18 +219,14 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
requests: RequestMetrics::default(),
attempts: AttemptMetrics::default(),
totals: Default::default(),
provider_accounts: AccountPoolMetrics {
total: 807,
enabled: 804,
unavailable: 792,
active: 15,
total: 1002,
normal: 943,
quota_exhausted: 52,
rate_limited: 0,
expired: 2,
invalid: 0,
quota_exhausted: 785,
refreshing: None,
disabled: 3,
banned: 2,
error: 4,
},
trend: Vec::new(),
account_usage: Vec::new(),
@@ -263,11 +259,160 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
&value["data"]["cards"]["credentials"]["unavailableValue"],
),
(
&serde_json::json!(807),
&serde_json::json!(15),
&serde_json::json!(792)
&serde_json::json!(1002),
&serde_json::json!(943),
&serde_json::json!(59)
),
);
assert_eq!(
(
&value["data"]["cards"]["credentials"]["total"],
&value["data"]["cards"]["credentials"]["available"],
&value["data"]["cards"]["credentials"]["unavailable"],
),
(
&serde_json::json!("1002"),
&serde_json::json!("943"),
&serde_json::json!("59")
),
);
}
#[tokio::test]
async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, AttemptMetrics, DashboardObservation, DashboardTotals, RequestMetrics,
TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
let range_requests = RequestMetrics {
request_count: 2,
input_tokens: 8,
cached_tokens: 2,
total_tokens: 12,
..RequestMetrics::default()
};
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
requests: range_requests,
attempts: AttemptMetrics::default(),
totals: DashboardTotals {
request_count: 42,
input_tokens: 800,
cached_tokens: 200,
total_tokens: 840,
billing_usd: None,
},
provider_accounts: AccountPoolMetrics::default(),
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_dashboard_lifetime_totals")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("dashboard response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
assert_eq!(status, StatusCode::OK, "{value}");
assert_eq!(
(
&value["data"]["cards"]["traffic"]["totalRequests"],
&value["data"]["cards"]["tokens"]["totalTokens"],
&value["data"]["cards"]["cache"]["totalHitRate"],
&value["data"]["cards"]["cache"]["totalCachedTokens"],
),
(
&serde_json::json!("42"),
&serde_json::json!("840"),
&serde_json::json!("25.0%"),
&serde_json::json!("200"),
),
);
}
#[tokio::test]
async fn dashboard_summary_should_default_to_current_china_day() {
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
china_day_start,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
requests: RequestMetrics::default(),
attempts: AttemptMetrics::default(),
totals: Default::default(),
provider_accounts: AccountPoolMetrics::default(),
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_dashboard_today_range")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let range = fixture
.dashboard_summary_range
.lock()
.expect("dashboard summary range")
.take()
.expect("recorded dashboard summary range");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(range.start, china_day_start(range.end));
}
#[tokio::test]
@@ -541,11 +686,14 @@ fn usage_record_with_account(
id: id.to_owned(),
client_api_key_ref: "key_detail".to_owned(),
config_revision: 1,
routing_scope: "all".to_owned(),
routing_group_refs: Vec::new(),
routing_group_names_snapshot: Vec::new(),
protocol: "openai".to_owned(),
operation: "responses".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "coding".to_owned(),
requested_model_id: Some("coding".to_owned()),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some(account_id.to_owned()),
provider_account_name: Some(name.to_owned()),
@@ -667,11 +815,14 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
id: "request_endpoint".to_owned(),
client_api_key_ref: "key_endpoint".to_owned(),
config_revision: 1,
routing_scope: "groups".to_owned(),
routing_group_refs: vec!["grp_usage".to_owned()],
routing_group_names_snapshot: vec!["Usage group".to_owned()],
protocol: "openai".to_owned(),
operation: "generate".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "grok-4.5".to_owned(),
requested_model_id: Some("grok-4.5".to_owned()),
provider_kind: Some("xai".to_owned()),
provider_account_ref: Some("acct_snapshot".to_owned()),
provider_account_name: Some("Snapshot Alpha".to_owned()),
@@ -786,6 +937,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
assert_eq!(
serde_json::json!({
"route": value["data"]["items"][0]["route"],
"routingScope": value["data"]["items"][0]["routingScope"],
"routingGroupRefs": value["data"]["items"][0]["routingGroupRefs"],
"routingGroupNamesSnapshot": value["data"]["items"][0]["routingGroupNamesSnapshot"],
"serviceTier": value["data"]["items"][0]["serviceTier"],
"accountId": value["data"]["items"][0]["accountId"],
"accountName": value["data"]["items"][0]["accountName"],
@@ -816,6 +970,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
}),
serde_json::json!({
"route": "/v1/responses",
"routingScope": "groups",
"routingGroupRefs": ["grp_usage"],
"routingGroupNamesSnapshot": ["Usage group"],
"serviceTier": "priority",
"accountId": "acct_snapshot",
"accountName": "Snapshot Alpha",
@@ -33,6 +33,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
let mut actual = rust_files(&root.join("src"));
actual.sort();
let mut expected = vec![
"src/admin/account_groups.rs",
"src/admin/accounts.rs",
"src/admin/auth.rs",
"src/admin/backups.rs",
@@ -51,6 +52,8 @@ fn source_tree_should_match_frozen_machine_manifest() {
"src/lib.rs",
"src/openai/auth.rs",
"src/openai/error.rs",
"src/openai/images/http.rs",
"src/openai/images/mod.rs",
"src/openai/mod.rs",
"src/openai/models.rs",
"src/openai/responses/error.rs",
@@ -73,6 +76,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
let mut actual = rust_files(&root.join("tests"));
actual.sort();
let mut expected = vec![
"tests/admin/account_groups.rs",
"tests/admin/accounts.rs",
"tests/admin/auth.rs",
"tests/admin/client_keys.rs",
@@ -88,6 +92,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
"tests/main.rs",
"tests/openai/auth.rs",
"tests/openai/error.rs",
"tests/openai/images.rs",
"tests/openai/mod.rs",
"tests/openai/models.rs",
"tests/openai/responses/http.rs",
+79 -1
View File
@@ -8,10 +8,12 @@ use futures::future::BoxFuture;
use gateway_core::{
engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ExecutionService, StartExecution,
StartedExecution,
StartProviderExecution, StartedExecution,
},
error::GatewayError,
health::{WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState},
routing::PublicModelId,
task::{WorkerId, WorkerKind},
};
use tower::ServiceExt;
@@ -31,6 +33,75 @@ async fn healthz_should_return_no_content_when_all_inputs_are_healthy() {
assert_eq!(response.status(), StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn healthz_should_ignore_provider_maintenance_worker_failure() {
let worker_health = StaticWorkerHealth(vec![worker_snapshot(
WorkerKind::QuotaCatalogHealth,
WorkerRuntimeState::BackingOff,
)]);
let response = crate::openai::api_router_with_worker_health(
Arc::new(UnusedExecution),
Arc::new(worker_health),
)
.await
.oneshot(
Request::builder()
.uri("/healthz")
.body(Body::empty())
.expect("health request"),
)
.await
.expect("health response");
assert_eq!(response.status(), StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn healthz_should_reject_critical_worker_failure() {
let worker_health = StaticWorkerHealth(vec![worker_snapshot(
WorkerKind::RuntimeSnapshotReconciliation,
WorkerRuntimeState::BackingOff,
)]);
let response = crate::openai::api_router_with_worker_health(
Arc::new(UnusedExecution),
Arc::new(worker_health),
)
.await
.oneshot(
Request::builder()
.uri("/healthz")
.body(Body::empty())
.expect("health request"),
)
.await
.expect("health response");
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
}
#[derive(Clone)]
struct StaticWorkerHealth(Vec<WorkerHealthSnapshot>);
impl WorkerHealthSource for StaticWorkerHealth {
fn snapshot(&self) -> Vec<WorkerHealthSnapshot> {
self.0.clone()
}
}
fn worker_snapshot(kind: WorkerKind, state: WorkerRuntimeState) -> WorkerHealthSnapshot {
let id = WorkerId::try_new(kind, "health-test").expect("worker ID");
WorkerHealthSnapshot {
key: WorkerHealthKey::Task(id),
state,
consecutive_failures: 1,
completed_cycles: 0,
last_fencing_token: None,
last_success_at: None,
last_failure_at: None,
last_error: Some("test worker failure".to_owned()),
}
}
struct UnusedExecution;
impl ExecutionService for UnusedExecution {
@@ -49,4 +120,11 @@ impl ExecutionService for UnusedExecution {
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async { unreachable!("health check does not execute requests") })
}
fn start_provider_endpoint(
&self,
_: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async { unreachable!("health check does not execute provider endpoints") })
}
}
@@ -0,0 +1,331 @@
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
};
use std::time::SystemTime;
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header::AUTHORIZATION},
};
use bytes::Bytes;
use futures::future::BoxFuture;
use gateway_core::engine::UpstreamSendState;
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionService,
ExecutionSession, StartExecution, StartProviderExecution, StartedExecution,
};
use gateway_core::engine::{CoordinatedEvent, EngineError, ModelRequestId};
use gateway_core::error::{
ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
};
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
use gateway_core::operation::{ImageRequestKind, Operation};
use gateway_core::routing::PublicModelId;
use serde_json::{Value, json};
use tower::ServiceExt;
use super::{api_router, authenticated_client};
const IMAGE_RESPONSE: &[u8] =
br#"{ "created": 1787212800, "data": [{"b64_json":"AAEC"}], "future": 9007199254740993 }"#;
#[derive(Debug, Clone, PartialEq)]
struct CapturedImageRequest {
provider: String,
endpoint: String,
transport: ClientTransport,
kind: ImageRequestKind,
body: Bytes,
context: Value,
}
struct ImageExecution {
client: AuthenticatedClient,
captured: Arc<Mutex<Vec<CapturedImageRequest>>>,
committed_statuses: Arc<Mutex<Vec<u16>>>,
fail_with_upstream_response: bool,
}
impl ImageExecution {
fn new() -> Arc<Self> {
Arc::new(Self {
client: authenticated_client("sk_images_test"),
captured: Arc::new(Mutex::new(Vec::new())),
committed_statuses: Arc::new(Mutex::new(Vec::new())),
fail_with_upstream_response: false,
})
}
fn failing() -> Arc<Self> {
Arc::new(Self {
client: authenticated_client("sk_images_test"),
captured: Arc::new(Mutex::new(Vec::new())),
committed_statuses: Arc::new(Mutex::new(Vec::new())),
fail_with_upstream_response: true,
})
}
fn captured(&self) -> Vec<CapturedImageRequest> {
self.captured.lock().expect("capture lock").clone()
}
fn committed_statuses(&self) -> Vec<u16> {
self.committed_statuses
.lock()
.expect("committed status lock")
.clone()
}
}
impl ExecutionService for ImageExecution {
fn authenticate(
&self,
plaintext: &str,
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
(plaintext == "sk_images_test")
.then(|| self.client.clone())
.ok_or(ClientAuthenticationError::InvalidKey)
}
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
Vec::new()
}
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
false
}
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async {
Err(GatewayError::new(
GatewayErrorKind::Internal,
"images test must use provider endpoint execution",
))
})
}
fn start_provider_endpoint(
&self,
request: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async move {
let Operation::GenerateImage(image) = &request.operation else {
return Err(GatewayError::new(
GatewayErrorKind::Internal,
"images test received a non-image operation",
));
};
let payload = image.payload();
self.captured
.lock()
.expect("capture lock")
.push(CapturedImageRequest {
provider: request.provider.as_str().to_owned(),
endpoint: request.metadata.endpoint,
transport: request.metadata.transport,
kind: image.kind(),
body: payload.body().clone(),
context: Value::Object(payload.context().clone()),
});
Ok(StartedExecution {
request_id: ModelRequestId::new("req_images_test").expect("request ID"),
created_at: SystemTime::now(),
stream: false,
session: Box::new(ImageSession {
response: Some(Bytes::from_static(IMAGE_RESPONSE)),
response_headers: vec![ProviderResponseHeader::new(
"x-image-rate-limit",
Bytes::from_static(b"42"),
)],
committed_statuses: Arc::clone(&self.committed_statuses),
finalized: Arc::new(AtomicBool::new(false)),
fail_with_upstream_response: self.fail_with_upstream_response,
}),
})
})
}
}
struct ImageSession {
response: Option<Bytes>,
response_headers: Vec<ProviderResponseHeader>,
committed_statuses: Arc<Mutex<Vec<u16>>>,
finalized: Arc<AtomicBool>,
fail_with_upstream_response: bool,
}
impl ExecutionSession for ImageSession {
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
Box::pin(async { unreachable!("buffered image delivery does not stream") })
}
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
Box::pin(async move {
if self.fail_with_upstream_response {
return Err(EngineError::Provider(
ProviderError::new(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::Sent,
)
.with_status(StatusCode::UNPROCESSABLE_ENTITY.as_u16())
.with_client_visible_upstream_response(
ClientVisibleUpstreamResponse::new(
StatusCode::UNPROCESSABLE_ENTITY.as_u16(),
Some(b"application/problem+json".to_vec()),
Bytes::from_static(
br#"{ "error":{"message":"future image validation"}, "future":9007199254740993 }"#,
),
)
.with_headers(vec![ProviderResponseHeader::new(
"x-future-image-error",
Bytes::from_static(b"preserved"),
)]),
),
));
}
Ok(vec![ProviderEvent::wire(
ProtocolWireEvent::raw_json(
"openai",
self.response.take().expect("single image response"),
)
.expect("OpenAI protocol"),
)])
})
}
fn response_headers(&self) -> &[ProviderResponseHeader] {
&self.response_headers
}
fn response_status_code(&self) -> Option<u16> {
Some(StatusCode::CREATED.as_u16())
}
fn commit_downstream(
&mut self,
client_status_code: Option<u16>,
) -> BoxFuture<'_, Result<(), EngineError>> {
Box::pin(async move {
self.committed_statuses
.lock()
.expect("committed status lock")
.push(client_status_code.expect("image response status"));
self.finalized.store(true, Ordering::Release);
Ok(())
})
}
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
Box::pin(async { Ok(()) })
}
fn is_finalized(&self) -> bool {
self.finalized.load(Ordering::Acquire)
}
fn cancel(&self) {
self.finalized.store(true, Ordering::Release);
}
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
Box::pin(async move {
self.finalized.store(true, Ordering::Release);
})
}
}
#[tokio::test]
async fn image_routes_should_not_decode_bodies_and_should_preserve_both_directions() {
let execution = ImageExecution::new();
let router = api_router(execution.clone()).await;
let cases = [
(
"/v1/images/generations",
ImageRequestKind::Generation,
br#"{ "model":"gpt-image-future", "prompt":"a lighthouse", "future":9007199254740993 }"#.as_slice(),
),
(
"/v1/images/edits",
ImageRequestKind::Edit,
br#"{"model":"gpt-image-2","images":[{"image_url":"data:image/png;base64,AAEC"}],"prompt":"add fog","prompt":"duplicate stays raw"}"#.as_slice(),
),
];
for (path, _, body) in &cases {
let response = router
.clone()
.oneshot(
Request::post(*path)
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", "application/json")
.header("x-codex-image-turn-id", "turn_image_route")
.body(Body::from(body.to_vec()))
.expect("image request"),
)
.await
.expect("image response");
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get("x-image-rate-limit"),
Some(&"42".parse().expect("header value"))
);
let response_body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read image response");
assert_eq!(response_body.as_ref(), IMAGE_RESPONSE);
}
let captured = execution.captured();
assert_eq!(captured.len(), cases.len());
for (captured, (endpoint, kind, body)) in captured.iter().zip(cases.iter()) {
assert_eq!(captured.provider, "openai");
assert_eq!(captured.endpoint, *endpoint);
assert_eq!(captured.transport, ClientTransport::HttpJson);
assert_eq!(captured.kind, *kind);
assert_eq!(captured.body.as_ref(), *body);
assert_eq!(
captured.context,
json!({"image_turn_id": "turn_image_route"})
);
}
assert_eq!(
execution.committed_statuses(),
vec![StatusCode::CREATED.as_u16(); cases.len()]
);
}
#[tokio::test]
async fn image_route_should_return_the_exact_upstream_error_response() {
let execution = ImageExecution::failing();
let response = api_router(execution)
.await
.oneshot(
Request::post("/v1/images/edits")
.header(AUTHORIZATION, "Bearer sk_images_test")
.header("content-type", "application/json")
.body(Body::from(
br#"{ "model":"gpt-image-2", "future_invalid":9007199254740993 }"#.to_vec(),
))
.expect("image request"),
)
.await
.expect("image response");
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
assert_eq!(
response.headers().get("content-type"),
Some(&"application/problem+json".parse().expect("content type"))
);
assert_eq!(
response.headers().get("x-future-image-error"),
Some(&"preserved".parse().expect("header value"))
);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read image response");
assert_eq!(
body.as_ref(),
br#"{ "error":{"message":"future image validation"}, "future":9007199254740993 }"#
);
}
+38 -6
View File
@@ -1,10 +1,11 @@
mod auth;
mod error;
mod images;
mod models;
mod responses;
mod router;
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
@@ -18,7 +19,9 @@ use gateway_core::engine::admission::{
use gateway_core::engine::continuation::{
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
};
use gateway_core::engine::credential::{AccountSelectionPolicy, RotationStrategy};
use gateway_core::engine::credential::{
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
};
use gateway_core::engine::execution::{
AuthenticatedClient, ClientApiKeyUsageSink, DefaultExecutionService, ExecutionService,
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
@@ -34,17 +37,37 @@ use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLif
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
use gateway_core::routing::{
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, RuntimeSnapshot,
UpstreamModelId,
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ProviderKind,
ProviderModel, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot, UpstreamModelId,
};
pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Router {
api_router_with_origins(execution, Vec::new()).await
}
pub(super) async fn api_router_with_worker_health(
execution: Arc<dyn ExecutionService>,
worker_health: Arc<dyn WorkerHealthSource>,
) -> axum::Router {
api_router_with_origins_and_worker_health(execution, Vec::new(), worker_health).await
}
pub(super) async fn api_router_with_origins(
execution: Arc<dyn ExecutionService>,
cors_allowed_origins: Vec<String>,
) -> axum::Router {
api_router_with_origins_and_worker_health(
execution,
cors_allowed_origins,
Arc::new(EmptyWorkerHealth),
)
.await
}
async fn api_router_with_origins_and_worker_health(
execution: Arc<dyn ExecutionService>,
cors_allowed_origins: Vec<String>,
worker_health: Arc<dyn WorkerHealthSource>,
) -> axum::Router {
let admin = crate::admin::AdminTestFixture::new().await;
gateway_api::initialize(
@@ -57,7 +80,7 @@ pub(super) async fn api_router_with_origins(
execution,
admin.services,
Vec::new(),
Arc::new(EmptyWorkerHealth),
worker_health,
Arc::new(TestLifecycle::default()),
)
.expect("API bundle")
@@ -94,6 +117,10 @@ impl ClientApiKeyUsageSink for IgnoredClientApiKeyUsage {
fn snapshot(plaintext: &str, provider_name: &str) -> RuntimeSnapshot {
let provider = ProviderKind::new(provider_name).expect("provider");
let account_directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
ProviderAccountId::new("acct_api_test").expect("account ID"),
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
)])));
let capabilities = ModelCapabilities::new(
BTreeSet::from([gateway_core::operation::OperationKind::Generate]),
Some(16_000),
@@ -119,12 +146,16 @@ fn snapshot(plaintext: &str, provider_name: &str) -> RuntimeSnapshot {
vec![ClientPolicy::new(
ClientApiKeyId::new("key_api_test").expect("key ID"),
PlaintextClientApiKey::new(plaintext).expect("plaintext key"),
provider,
Arc::new(FrozenAccountScope::new(
Arc::clone(&account_directory),
ClientRoutingScope::all_accounts(),
)),
true,
RateLimits::unlimited(),
)],
)
.expect("runtime snapshot")
.with_account_directory(account_directory)
}
#[derive(Default)]
@@ -260,6 +291,7 @@ struct UnusedContinuation;
impl NativeContinuationPort for UnusedContinuation {
fn resolve<'a>(
&'a self,
_: &'a ClientApiKeyId,
_: &'a PreviousResponseId,
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
Box::pin(async { unreachable!("authentication fixture does not execute") })
@@ -7,7 +7,7 @@ use axum::{
use futures::future::BoxFuture;
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ExecutionService, StartExecution,
StartedExecution,
StartProviderExecution, StartedExecution,
};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::routing::{
@@ -99,6 +99,18 @@ impl ExecutionService for ModelsExecution {
))
})
}
fn start_provider_endpoint(
&self,
_: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async {
Err(GatewayError::new(
GatewayErrorKind::Internal,
"models test must not start a provider endpoint",
))
})
}
}
fn authorized_request(path: &str) -> Request<Body> {
@@ -17,7 +17,7 @@ use bytes::Bytes;
use futures::future::{BoxFuture, pending};
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
StartExecution, StartedExecution,
StartExecution, StartProviderExecution, StartedExecution,
};
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
use gateway_core::error::{
@@ -190,6 +190,18 @@ impl ExecutionService for ContextCaptureExecution {
))
})
}
fn start_provider_endpoint(
&self,
_: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async {
Err(GatewayError::new(
GatewayErrorKind::Internal,
"responses context test must not start a provider endpoint",
))
})
}
}
impl FakeSession {
@@ -788,6 +800,10 @@ async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresenta
let trace = Arc::new(Trace::default());
let headers = vec![
ProviderResponseHeader::new("x-models-etag", Bytes::from_static(b"models-v2")),
ProviderResponseHeader::new(
"x-codex-turn-state",
Bytes::from_static(b"turn-state-from-upstream"),
),
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
@@ -834,6 +850,13 @@ async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresenta
.map(HeaderValue::as_bytes),
Some(b"\xffopaque".as_slice())
);
assert_eq!(
response
.headers()
.get("x-codex-turn-state")
.and_then(|value| value.to_str().ok()),
Some("turn-state-from-upstream")
);
assert!(response.headers().get("authorization").is_none());
assert!(response.headers().get("connection").is_none());
assert!(response.headers().get("x-hop-secret").is_none());
@@ -130,7 +130,7 @@ fn decoder_should_preserve_opaque_client_model_values() {
"input": "hello"
}));
assert_eq!(decoded.metadata().public_model(), model);
assert_eq!(decoded.metadata().requested_model(), model);
assert_eq!(
openai_wire_body(&decoded)
.get("model")
@@ -451,15 +451,23 @@ fn decoder_should_not_reject_large_bodies_using_catalog_context_limits() {
.to_string();
let decoded = decode_request(body.as_bytes()).expect("large request should decode");
let plan = super::snapshot("sk_context_test", "openai")
let snapshot = super::snapshot("sk_context_test", "openai");
let plan = snapshot
.plan(
&PublicModelId::new(decoded.metadata().public_model()).expect("public model"),
&PublicModelId::new(decoded.metadata().requested_model()).expect("public model"),
decoded.operation(),
snapshot.all_account_scope(),
&RoutingContext::default(),
)
.expect("large body must not be locally context-gated");
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "model-a");
assert_eq!(
plan.candidates()[0]
.upstream_model()
.expect("model route candidate")
.as_str(),
"model-a"
);
}
#[test]
@@ -13,7 +13,7 @@ use futures::{SinkExt, StreamExt, future::BoxFuture};
use gateway_api::openai::responses::ResponseCreateFrameError;
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
StartExecution, StartedExecution,
StartExecution, StartProviderExecution, StartedExecution,
};
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
use gateway_core::error::{GatewayError, ProviderError, ProviderErrorKind};
@@ -271,6 +271,13 @@ impl ExecutionService for AtomicFailureExecution {
})
})
}
fn start_provider_endpoint(
&self,
_: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async { unreachable!("WebSocket test does not execute provider endpoints") })
}
}
fn atomic_failure_batch() -> CoordinatedEvent {
@@ -411,6 +418,10 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
ProviderResponseHeader::new("x-future-header", Bytes::from_static(b"future-value")),
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
ProviderResponseHeader::new(
"x-codex-turn-state",
Bytes::from_static(b"turn-state-from-upstream"),
),
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
ProviderResponseHeader::new(
@@ -483,6 +494,10 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
// 官方 metadata 是 string map,同名多值只能保持既有的后值覆盖语义。
assert_eq!(headers.get("x-future-multi"), Some(&json!("second")));
assert_eq!(headers.get("x-request-id"), Some(&json!("req_upstream")));
assert_eq!(
headers.get("x-codex-turn-state"),
Some(&json!("turn-state-from-upstream"))
);
for omitted in [
"x-future-bytes",
"bad\0name",
@@ -9,6 +9,7 @@ use futures::future::BoxFuture;
use crate::engine::credential::ProviderAccountId;
use crate::operation::ProviderSessionState;
use crate::policy::ClientApiKeyId;
use crate::routing::ProviderKind;
/// 客户端或 Provider 传递的 opaque response handle。
@@ -48,14 +49,15 @@ pub enum NativeContinuationScope {
/// 从可丢失会话亲和存储恢复的 native previous-response pin。
///
/// 该值不依赖调用方 API Key;它只阻止 native handle 与不透明 Provider state
/// 被发送到错误的 Provider 或账号。
/// 该值同时绑定调用方 API Key、Provider 与账号,阻止 native handle 和不透明
/// Provider state 被跨租户或跨上游目标复用。
#[derive(Clone, PartialEq)]
pub struct NativeContinuationPin {
/// 客户端提交、用于查找可丢失会话亲和记录的 response ID。
previous_response_id: PreviousResponseId,
/// Provider 原生 response handle。
upstream_response_id: PreviousResponseId,
client_api_key_id: ClientApiKeyId,
provider: ProviderKind,
account: ProviderAccountId,
scope: NativeContinuationScope,
@@ -67,12 +69,14 @@ impl NativeContinuationPin {
pub const fn new(
previous_response_id: PreviousResponseId,
upstream_response_id: PreviousResponseId,
client_api_key_id: ClientApiKeyId,
provider: ProviderKind,
account: ProviderAccountId,
) -> Self {
Self {
previous_response_id,
upstream_response_id,
client_api_key_id,
provider,
account,
scope: NativeContinuationScope::ConnectionLocal,
@@ -105,6 +109,16 @@ impl NativeContinuationPin {
&self.upstream_response_id
}
#[must_use]
pub const fn client_api_key_id(&self) -> &ClientApiKeyId {
&self.client_api_key_id
}
#[must_use]
pub fn matches_client(&self, client_api_key_id: &ClientApiKeyId) -> bool {
self.client_api_key_id == *client_api_key_id
}
#[must_use]
pub const fn provider(&self) -> &ProviderKind {
&self.provider
@@ -139,6 +153,7 @@ impl fmt::Debug for NativeContinuationPin {
.debug_struct("NativeContinuationPin")
.field("previous_response_id", &"<redacted>")
.field("upstream_response_id", &"<redacted>")
.field("client_api_key_id", &self.client_api_key_id)
.field("provider", &self.provider)
.field("account", &self.account)
.field("scope", &self.scope)
@@ -190,21 +205,57 @@ impl fmt::Debug for ContinuationBinding {
}
}
/// Native continuation lookup 的稳定失败分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum NativeContinuationStoreErrorKind {
/// Redis 等可丢失协调存储暂不可用;调用方可以按外部 continuation fail-open。
Unavailable,
/// 存储中的记录无法还原为可信 pin;不得将原始 handle 继续透传上游。
InvalidData,
/// 已存在的记录属于另一个客户端 API Key;必须 fail closed。
OwnershipMismatch,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error("native continuation store is unavailable: {detail}")]
#[error("native continuation store failed: {kind:?}: {detail}")]
pub struct NativeContinuationStoreError {
kind: NativeContinuationStoreErrorKind,
detail: String,
}
impl NativeContinuationStoreError {
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
/// 构造可 fail-open 的协调存储不可用错误;`detail` 不应包含 response handle。
#[must_use]
pub fn new(detail: impl Into<String>) -> Self {
pub fn unavailable(detail: impl Into<String>) -> Self {
Self {
kind: NativeContinuationStoreErrorKind::Unavailable,
detail: detail.into(),
}
}
/// 构造必须 fail closed 的无效记录错误;`detail` 不应包含 response handle。
#[must_use]
pub fn invalid_data(detail: impl Into<String>) -> Self {
Self {
kind: NativeContinuationStoreErrorKind::InvalidData,
detail: detail.into(),
}
}
/// 构造不泄露记录归属细节的跨调用方错误。
#[must_use]
pub fn ownership_mismatch() -> Self {
Self {
kind: NativeContinuationStoreErrorKind::OwnershipMismatch,
detail: "continuation belongs to a different client API key".to_owned(),
}
}
#[must_use]
pub const fn kind(&self) -> NativeContinuationStoreErrorKind {
self.kind
}
#[must_use]
pub fn detail(&self) -> &str {
&self.detail
@@ -213,11 +264,12 @@ impl NativeContinuationStoreError {
/// 可丢失的 previous-response 亲和存储端口。
///
/// 亲和记录不按调用方 API Key 隔离:与旧版一样,response ID 是会话级 opaque
/// handle;Redis 不可用或超时必须由调用方 fail-open。
/// 亲和记录按调用方 API Key 隔离;Redis 不可用或超时必须由调用方 fail-open,
/// 但命中其他 Key 的记录必须 fail closed。
pub trait NativeContinuationPort: Send + Sync {
fn resolve<'a>(
&'a self,
client_api_key_id: &'a ClientApiKeyId,
previous_response_id: &'a PreviousResponseId,
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>>;
@@ -46,6 +46,10 @@ impl AccountSelection {
Self::Diagnostic(account) => Some(account),
}
}
const fn is_diagnostic(&self) -> bool {
matches!(self, Self::Diagnostic(_))
}
}
impl<S: ?Sized> AttemptCoordinator<S>
@@ -126,6 +130,15 @@ where
.map(ProviderAccountStateOwner::from_continuation);
let continuation_attempt =
initial_continuation_attempt(&operation, &plan, continuation.as_ref());
let candidate_index = continuation
.as_ref()
.and_then(ContinuationBinding::pinned)
.map_or(Ok(0), |pin| {
plan.candidates()
.iter()
.position(|candidate| candidate.provider() == pin.provider())
.ok_or(EngineError::ContinuationPinMismatch)
})?;
let image_generation_requested = operation.image_generation_requested();
let mut session = ResponseExecutionSession {
engine: Arc::clone(&self.engine),
@@ -149,6 +162,7 @@ where
account_state_owner,
cancellation,
attempts: 0,
candidate_index,
excluded_accounts: BTreeSet::new(),
credential_recovery_attempted_accounts: BTreeSet::new(),
recovery_account: None,
@@ -225,6 +239,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
account_state_owner: Option<ProviderAccountStateOwner>,
cancellation: CancellationToken,
attempts: u32,
candidate_index: usize,
excluded_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
credential_recovery_attempted_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
@@ -440,6 +455,15 @@ where
.unwrap_or_default()
}
/// 返回最终选中 attempt 观察到的上游 HTTP 状态码。
#[must_use]
pub fn response_status_code(&self) -> Option<u16> {
self.current
.as_ref()
.and_then(|current| current.response_observation.as_ref())
.and_then(ProviderResponseObservation::status_code)
}
/// 将已完成响应的账号事实与 Provider 私有状态封装为可丢失的亲和记录。
///
/// Core 不读取 `state` 内容;Provider 将在后续同账号 continuation 时自行解释。
@@ -461,6 +485,7 @@ where
NativeContinuationPin::new(
previous_response_id,
upstream_response_id,
self.client_api_key_ref.clone(),
provider,
account,
)
@@ -565,7 +590,7 @@ where
if self.attempts >= self.plan.max_attempts().get() {
return Err(EngineError::EmptyRoutingPlan);
}
let Some(candidate) = self.plan.candidates().first().cloned() else {
let Some(candidate) = self.plan.candidates().get(self.candidate_index).cloned() else {
let error = GatewayError::new(
GatewayErrorKind::NoAvailableProvider,
"no upstream Provider is available",
@@ -606,7 +631,8 @@ where
self.excluded_accounts.clone(),
pinned_account.clone(),
self.account_state_owner.clone(),
),
)
.with_account_scope(Arc::clone(self.plan.account_scope())),
}
.with_credential_recovery_attempted(pinned_account.as_ref().is_some_and(|account| {
self.credential_recovery_attempted_accounts
@@ -660,6 +686,23 @@ where
ProviderBoundary::Result(result) => match *result {
Ok(stream) => stream,
Err(error) => {
if self.prepare_unavailable_native_continuation_replay(&error) {
return Ok(Some(PullOutcome::AttemptDiscarded));
}
if matches!(
error.kind(),
ProviderErrorKind::AccountCapacityUnavailable
| ProviderErrorKind::NoEligibleAccount
| ProviderErrorKind::ProviderInfrastructureUnavailable
) && error.send_state() == UpstreamSendState::NotSent
&& matches!(
self.continuation_attempt,
ContinuationAttempt::None | ContinuationAttempt::ReplayAny
)
&& self.advance_provider_candidate()
{
return Ok(Some(PullOutcome::AttemptDiscarded));
}
if matches!(
error.kind(),
ProviderErrorKind::AccountCapacityUnavailable
@@ -713,6 +756,27 @@ where
}
let metadata = stream.metadata().clone();
if !self.account_selection.is_diagnostic()
&& metadata
.provider_account_id()
.is_none_or(|account| !candidate.account_scope().allows(account))
{
let error = GatewayError::new(
GatewayErrorKind::Internal,
"provider selected an account outside the frozen client scope",
);
self.finish_failure(FailureFinalization {
outcome: ExecutionOutcome::Failed,
send_state: self.current_send_state(),
error,
upstream_status_code: None,
provider_error_code: None,
retry_after_ms: None,
provider_response_id: None,
})
.await?;
return Err(EngineError::AccountOutsideClientScope);
}
if pinned_account
.as_ref()
.is_some_and(|required| metadata.provider_account_id() != Some(required))
@@ -773,7 +837,7 @@ where
provider_kind: metadata.provider().clone(),
provider_account_id: metadata.provider_account_id().cloned(),
provider_account_ref: metadata.provider_account_id().cloned(),
upstream_model_id: metadata.upstream_model().clone(),
upstream_model_id: metadata.upstream_model().cloned(),
upstream_transport: metadata.transport().as_str().to_owned(),
http_version: None,
};
@@ -816,6 +880,17 @@ where
Ok(None)
}
fn advance_provider_candidate(&mut self) -> bool {
let Some(next) = self.candidate_index.checked_add(1) else {
return false;
};
if next >= self.plan.candidates().len() {
return false;
}
self.candidate_index = next;
true
}
async fn observe_event(&mut self, event: &GatewayEvent) {
self.observe_event_facts(event);
self.mark_send_observed().await;
@@ -1008,7 +1083,7 @@ where
trigger: current.trigger,
provider_kind: current.metadata.provider().clone(),
account_id: current.metadata.provider_account_id().cloned(),
upstream_model_id: current.metadata.upstream_model().clone(),
upstream_model_id: current.metadata.upstream_model().cloned(),
upstream_status_code: current
.response_observation
.as_ref()
@@ -1096,6 +1171,42 @@ where
true
}
/// 原生续写的原账号在本地调度阶段已不可用时,交给 Provider 执行跨账号恢复。
///
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
/// 对应 Provider 的会话状态时才进入恢复;是否保留 native handle、执行 probe
/// 或使用完整 transcript,由 Provider 自己的协议边界决定。
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
if self.account_selection.required_account().is_some()
|| self.continuation_attempt != ContinuationAttempt::Native
|| !matches!(
error.kind(),
ProviderErrorKind::NoEligibleAccount
| ProviderErrorKind::AccountCapacityUnavailable
)
{
return false;
}
let Some(pin) = self
.continuation
.as_ref()
.and_then(ContinuationBinding::pinned)
else {
return false;
};
if self
.operation
.provider_session_state(pin.provider().as_str())
.is_none()
{
return false;
}
self.continuation_attempt = ContinuationAttempt::ReplayAny;
self.excluded_accounts.insert(pin.account().clone());
true
}
fn reset_uncommitted_observations(&mut self) {
self.usage = Usage::new();
self.cost = CostEstimate::unavailable();
@@ -1380,8 +1491,13 @@ where
}
fn image_generation_succeeded(&self) -> Option<bool> {
self.image_generation_requested
.then(|| self.usage.image_output_tokens.unwrap_or_default() > 0)
self.image_generation_requested.then(|| {
if matches!(self.operation, Operation::GenerateImage(_)) {
self.upstream_complete
} else {
self.usage.image_output_tokens.unwrap_or_default() > 0
}
})
}
fn current_send_state(&self) -> UpstreamSendState {
File diff suppressed because it is too large Load Diff
@@ -15,7 +15,8 @@ use crate::engine::admission::{
ClientAdmissionDecision, ClientAdmissionPort, ClientAdmissionRejection, ClientAdmissionRequest,
};
use crate::engine::continuation::{
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
ContinuationBinding, NativeContinuationPin, NativeContinuationPort,
NativeContinuationStoreErrorKind, PreviousResponseId,
};
use crate::engine::coordinator::ResponseExecutionSession;
use crate::engine::probe::{
@@ -117,6 +118,35 @@ pub struct StartExecution {
pub metadata: ExecutionRequestMetadata,
}
/// 启动一个由协议 adapter 明确绑定到 Provider 自有端点的请求。
pub struct StartProviderExecution {
pub client: AuthenticatedClient,
pub provider: ProviderKind,
pub operation: Operation,
pub metadata: ExecutionRequestMetadata,
}
enum ExecutionTarget {
Model(PublicModelId),
ProviderEndpoint(ProviderKind),
}
impl ExecutionTarget {
fn into_public_model(self) -> Option<PublicModelId> {
match self {
Self::Model(model) => Some(model),
Self::ProviderEndpoint(_) => None,
}
}
}
struct PendingStartExecution {
client: AuthenticatedClient,
target: ExecutionTarget,
operation: Operation,
metadata: ExecutionRequestMetadata,
}
pub struct StartedExecution {
pub request_id: ModelRequestId,
pub created_at: SystemTime,
@@ -128,6 +158,9 @@ pub trait ExecutionSession: Send {
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>>;
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>>;
fn response_headers(&self) -> &[ProviderResponseHeader];
fn response_status_code(&self) -> Option<u16> {
None
}
fn commit_downstream(
&mut self,
client_status_code: Option<u16>,
@@ -155,6 +188,10 @@ pub trait ExecutionService: Send + Sync {
&self,
request: StartExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
fn start_provider_endpoint(
&self,
request: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
}
/// 成功认证后的 API Key 使用事实接收器。
@@ -246,9 +283,44 @@ impl DefaultExecutionService {
}
}
async fn start_inner(
async fn start_inner(&self, request: StartExecution) -> Result<StartedExecution, GatewayError> {
let StartExecution {
client,
public_model,
operation,
metadata,
} = request;
self.start_inner_with_target(PendingStartExecution {
client,
target: ExecutionTarget::Model(public_model),
operation,
metadata,
})
.await
}
async fn start_provider_endpoint_inner(
&self,
mut request: StartExecution,
request: StartProviderExecution,
) -> Result<StartedExecution, GatewayError> {
let StartProviderExecution {
client,
provider,
operation,
metadata,
} = request;
self.start_inner_with_target(PendingStartExecution {
client,
target: ExecutionTarget::ProviderEndpoint(provider),
operation,
metadata,
})
.await
}
async fn start_inner_with_target(
&self,
mut request: PendingStartExecution,
) -> Result<StartedExecution, GatewayError> {
request.client.policy.authorize().map_err(|_| {
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
@@ -261,22 +333,40 @@ impl DefaultExecutionService {
})?;
let request_id = new_request_id()?;
let routing_context = self
.route_context(request.client.policy.provider_kind())
.route_context(request.client.policy.account_scope().provider_kinds())
.await?;
let plan = request
.client
.snapshot
.plan(&request.public_model, &request.operation, &routing_context)
.map_err(map_routing_error)?;
let account_scope = Arc::clone(request.client.policy.account_scope());
let plan = match &request.target {
ExecutionTarget::ProviderEndpoint(provider) => {
request.client.snapshot.plan_provider_endpoint(
provider,
&request.operation,
account_scope,
&routing_context,
)
}
ExecutionTarget::Model(public_model) => request.client.snapshot.plan(
public_model,
&request.operation,
account_scope,
&routing_context,
),
}
.map_err(map_routing_error)?;
let continuation = match request.metadata.previous_response_id.as_ref() {
Some(previous) => {
let resolve = self.continuation.resolve(previous).fuse();
let resolve = self
.continuation
.resolve(request.client.policy.key_id(), previous)
.fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(resolve, timeout);
let pin = select_biased! {
result = resolve => match result {
Ok(pin) => pin,
Err(error) => {
Err(error)
if error.kind() == NativeContinuationStoreErrorKind::Unavailable =>
{
tracing::debug!(
request_id = request_id.as_str(),
%error,
@@ -284,10 +374,53 @@ impl DefaultExecutionService {
);
None
}
Err(error)
if error.kind()
== NativeContinuationStoreErrorKind::OwnershipMismatch =>
{
return Err(GatewayError::new(
GatewayErrorKind::PolicyDenied,
"continuation does not belong to this client API key",
));
}
Err(error) => {
tracing::warn!(
request_id = request_id.as_str(),
%error,
"Continuation affinity 记录无效,已拒绝续接"
);
return Err(GatewayError::new(
GatewayErrorKind::Internal,
"continuation state is invalid",
));
}
},
_ = timeout => None,
};
match pin {
Some(pin) if !pin.matches_client(request.client.policy.key_id()) => {
return Err(GatewayError::new(
GatewayErrorKind::PolicyDenied,
"continuation does not belong to this client API key",
));
}
Some(pin) if !request.client.policy.account_scope().allows(pin.account()) => {
return Err(GatewayError::new(
GatewayErrorKind::PolicyDenied,
"continuation account is outside the client account scope",
));
}
Some(pin)
if !plan
.candidates()
.iter()
.any(|candidate| candidate.provider() == pin.provider()) =>
{
return Err(GatewayError::new(
GatewayErrorKind::NoAvailableProvider,
"continuation provider is not available",
));
}
Some(pin) => {
attach_continuation_session_state(&mut request.operation, &pin);
ContinuationBinding::Pinned(pin)
@@ -321,18 +454,24 @@ impl DefaultExecutionService {
async fn start_without_continuation(
&self,
request: StartExecution,
request: PendingStartExecution,
request_id: ModelRequestId,
started_at: SystemTime,
deadline_at: SystemTime,
plan: crate::routing::RoutingPlan,
continuation: Option<ContinuationBinding>,
) -> Result<StartedExecution, GatewayError> {
let PendingStartExecution {
client,
target,
operation,
metadata,
} = request;
let admission_request = ClientAdmissionRequest {
model_request_id: request_id.clone(),
client_api_key_id: request.client.policy.key_id().clone(),
client_api_key_id: client.policy.key_id().clone(),
lease_ttl: MODEL_REQUEST_DEADLINE,
limits: request.client.policy.limits(),
limits: client.policy.limits(),
};
match self
.admissions
@@ -357,30 +496,35 @@ impl DefaultExecutionService {
}
let admission = AdmissionLease {
port: Arc::clone(&self.admissions),
client_api_key_id: request.client.policy.key_id().clone(),
client_api_key_id: client.policy.key_id().clone(),
model_request_id: request_id.clone(),
};
let observation = self
.providers
.request_observation(request.client.policy.provider_kind(), &request.operation);
let observation = plan
.candidates()
.first()
.map_or_else(Default::default, |candidate| {
self.providers
.request_observation(candidate.provider(), &operation)
});
let new_request = NewModelRequest {
id: request_id.clone(),
client_api_key_id: Some(request.client.policy.key_id().clone()),
client_api_key_ref: request.client.policy.key_id().clone(),
client_api_key_id: Some(client.policy.key_id().clone()),
client_api_key_ref: client.policy.key_id().clone(),
config_revision: plan.config_revision(),
protocol: request.metadata.protocol,
operation: request.operation.kind(),
endpoint: request.metadata.endpoint,
client_transport: request.metadata.transport.as_str().to_owned(),
requested_model: request.public_model,
client_ip: request.metadata.client_ip,
user_agent: request.metadata.user_agent,
routing: client.policy.account_scope().routing_snapshot(),
protocol: metadata.protocol,
operation: operation.kind(),
endpoint: metadata.endpoint,
client_transport: metadata.transport.as_str().to_owned(),
requested_model: target.into_public_model(),
client_ip: metadata.client_ip,
user_agent: metadata.user_agent,
reasoning_effort: observation.reasoning_effort,
reasoning_preset: observation.reasoning_preset,
request_kind: observation.request_kind,
subagent_kind: observation.subagent_kind,
compact: observation.compact,
image_generation_requested: request.operation.image_generation_requested(),
image_generation_requested: operation.image_generation_requested(),
started_at,
deadline_at,
};
@@ -388,7 +532,7 @@ impl DefaultExecutionService {
.coordinator
.start(
new_request,
request.operation,
operation,
plan,
None,
continuation,
@@ -405,7 +549,7 @@ impl DefaultExecutionService {
Ok(StartedExecution {
request_id,
created_at: started_at,
stream: request.metadata.stream,
stream: metadata.stream,
session: Box::new(DefaultExecutionSession::new(
core,
admission,
@@ -417,38 +561,42 @@ impl DefaultExecutionService {
async fn route_context(
&self,
provider_kind: &ProviderKind,
provider_kinds: &BTreeSet<ProviderKind>,
) -> Result<RoutingContext, GatewayError> {
let decision = self.circuits.decision(provider_kind).fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(decision, timeout);
let decision = select_biased! {
result = decision => Some(result),
_ = timeout => None,
};
let blocked_providers = match decision {
Some(Ok(ProviderCircuitDecision::Allow)) => BTreeSet::new(),
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
BTreeSet::from([provider_kind.clone()])
let decisions = futures::future::join_all(provider_kinds.iter().map(|provider_kind| {
let circuits = Arc::clone(&self.circuits);
async move {
let decision = circuits.decision(provider_kind).fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(decision, timeout);
let decision = select_biased! {
result = decision => Some(result),
_ = timeout => None,
};
(provider_kind, decision)
}
Some(Err(error)) => {
tracing::warn!(
}))
.await;
let mut blocked_providers = BTreeSet::new();
for (provider_kind, decision) in decisions {
match decision {
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
blocked_providers.insert(provider_kind.clone());
}
Some(Err(error)) => tracing::warn!(
provider = provider_kind.as_str(),
%error,
"Provider circuit 读取失败,按可重建协调状态 fail-open"
);
BTreeSet::new()
}
None => {
tracing::warn!(
),
None => tracing::warn!(
provider = provider_kind.as_str(),
"Provider circuit 读取超时,按可重建协调状态 fail-open"
);
BTreeSet::new()
),
Some(Ok(ProviderCircuitDecision::Allow)) => {}
}
};
}
Ok(RoutingContext {
provider_kind: Some(provider_kind.clone()),
required_provider: None,
blocked_providers,
})
}
@@ -479,11 +627,16 @@ impl DefaultExecutionService {
GatewayError::new(GatewayErrorKind::Unsupported, "requested model is invalid")
})?;
let routing_context = RoutingContext {
provider_kind: Some(provider_kind),
required_provider: Some(provider_kind),
..RoutingContext::default()
};
let plan = snapshot
.plan(&public_model, &operation, &routing_context)
.plan(
&public_model,
&operation,
snapshot.all_account_scope(),
&routing_context,
)
.map_err(map_routing_error)?;
let started_at = SystemTime::now();
let deadline_at = started_at
@@ -499,11 +652,12 @@ impl DefaultExecutionService {
client_api_key_id: None,
client_api_key_ref: actor,
config_revision: plan.config_revision(),
routing: crate::routing::AccountRoutingSnapshot::all(),
protocol: "admin_connection_test".to_owned(),
operation: operation.kind(),
endpoint: "/api/admin/accounts/connection-test".to_owned(),
client_transport: ClientTransport::InternalProbe.as_str().to_owned(),
requested_model: public_model,
requested_model: Some(public_model),
client_ip: None,
user_agent: None,
reasoning_effort: None,
@@ -693,19 +847,19 @@ impl ExecutionService for DefaultExecutionService {
fn public_models(&self, client: &AuthenticatedClient) -> Vec<PublicModelId> {
client
.snapshot
.public_models_for_provider(client.policy.provider_kind())
.public_models_for_scope(client.policy.account_scope())
}
fn public_model_profiles(&self, client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
client
.snapshot
.public_model_profiles_for_provider(client.policy.provider_kind())
.public_model_profiles_for_scope(client.policy.account_scope())
}
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool {
client
.snapshot
.contains_public_model_for_provider(model, client.policy.provider_kind())
.contains_public_model_for_scope(model, client.policy.account_scope())
}
fn start(
@@ -714,6 +868,13 @@ impl ExecutionService for DefaultExecutionService {
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async move { self.start_inner(request).await })
}
fn start_provider_endpoint(
&self,
request: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async move { self.start_provider_endpoint_inner(request).await })
}
}
impl AccountProbe for DefaultExecutionService {
@@ -879,6 +1040,12 @@ impl ExecutionSession for DefaultExecutionSession {
.unwrap_or_default()
}
fn response_status_code(&self) -> Option<u16> {
self.core
.as_ref()
.and_then(ResponseExecutionSession::response_status_code)
}
fn commit_downstream(
&mut self,
client_status_code: Option<u16>,
@@ -1006,7 +1173,9 @@ fn new_request_id() -> Result<ModelRequestId, GatewayError> {
fn map_routing_error(error: crate::error::RoutingError) -> GatewayError {
match error {
crate::error::RoutingError::NoCapableProvider { .. } => GatewayError::new(
crate::error::RoutingError::NoCapableProvider { .. }
| crate::error::RoutingError::NoCapableProviderEndpoint { .. }
| crate::error::RoutingError::EmptyAccountScope => GatewayError::new(
GatewayErrorKind::NoAvailableProvider,
"no provider can execute this request",
),
+34 -5
View File
@@ -233,6 +233,7 @@ pub struct AccountAttemptContext {
state_owner: Option<ProviderAccountStateOwner>,
credential_recovery_attempted: bool,
diagnostic_required_account: bool,
account_scope: Option<Arc<crate::routing::FrozenAccountScope>>,
}
impl AccountAttemptContext {
@@ -248,6 +249,7 @@ impl AccountAttemptContext {
state_owner,
credential_recovery_attempted: false,
diagnostic_required_account: false,
account_scope: None,
}
}
@@ -266,9 +268,17 @@ impl AccountAttemptContext {
state_owner,
credential_recovery_attempted: false,
diagnostic_required_account: true,
account_scope: None,
}
}
/// 附着普通请求认证时冻结的账号范围。
#[must_use]
pub fn with_account_scope(mut self, scope: Arc<crate::routing::FrozenAccountScope>) -> Self {
self.account_scope = Some(scope);
self
}
/// 标记本请求已对即将选择的固定账号执行过一次凭据恢复。
#[must_use]
pub const fn with_credential_recovery_attempted(mut self, attempted: bool) -> Self {
@@ -301,6 +311,11 @@ impl AccountAttemptContext {
pub const fn is_diagnostic_required_account(&self) -> bool {
self.diagnostic_required_account
}
#[must_use]
pub const fn account_scope(&self) -> Option<&Arc<crate::routing::FrozenAccountScope>> {
self.account_scope.as_ref()
}
}
/// 请求中 Provider 账号绑定状态的唯一归属。
@@ -347,9 +362,14 @@ pub enum ContinuationAttempt {
None,
/// 使用 Store 解析出的原生 handle 与账号绑定。
Native,
/// 在原账号上用完整连接内 transcript 重放。
/// 在原账号上执行 Provider 定义的恢复。
///
/// 只有持有可携带 transcript 的 Provider 才能清除 native handle 后重放;
/// 只持有 opaque state 的 Provider 必须保留 continuation 依赖或返回客户端错误。
ReplayOwner,
/// transcript 已可携带,允许选择其他账号。
/// 允许选择其他账号并执行 Provider 定义的恢复。
///
/// 该枚举本身不证明客户端 input 完整,也不授权 Provider 清除 previous-response。
ReplayAny,
}
@@ -506,6 +526,12 @@ impl AttemptContext {
self.account.is_diagnostic_required_account()
}
/// 普通请求认证时冻结的账号范围;管理端诊断为 `None`。
#[must_use]
pub const fn account_scope(&self) -> Option<&Arc<crate::routing::FrozenAccountScope>> {
self.account.account_scope()
}
#[must_use]
pub const fn continuation(&self) -> Option<&ContinuationBinding> {
self.continuation.as_ref()
@@ -529,11 +555,12 @@ pub struct NewModelRequest {
pub client_api_key_id: Option<ClientApiKeyId>,
pub client_api_key_ref: ClientApiKeyId,
pub config_revision: ConfigRevision,
pub routing: crate::routing::AccountRoutingSnapshot,
pub protocol: String,
pub operation: OperationKind,
pub endpoint: String,
pub client_transport: String,
pub requested_model: PublicModelId,
pub requested_model: Option<PublicModelId>,
pub client_ip: Option<IpAddr>,
pub user_agent: Option<String>,
pub reasoning_effort: Option<String>,
@@ -555,7 +582,7 @@ pub struct AttemptRecord {
pub provider_kind: ProviderKind,
pub provider_account_id: Option<ProviderAccountId>,
pub provider_account_ref: Option<ProviderAccountId>,
pub upstream_model_id: UpstreamModelId,
pub upstream_model_id: Option<UpstreamModelId>,
pub upstream_transport: String,
pub http_version: Option<String>,
}
@@ -568,7 +595,7 @@ pub struct IntermediateFailure {
pub trigger: AttemptTrigger,
pub provider_kind: ProviderKind,
pub account_id: Option<ProviderAccountId>,
pub upstream_model_id: UpstreamModelId,
pub upstream_model_id: Option<UpstreamModelId>,
pub upstream_status_code: Option<u16>,
pub upstream_request_id: Option<String>,
pub error: ProviderError,
@@ -749,6 +776,8 @@ pub enum EngineError {
ContinuationPinMismatch,
#[error("provider did not use the account required by this execution")]
RequiredAccountMismatch,
#[error("provider selected an account outside the frozen client scope")]
AccountOutsideClientScope,
#[error("provider execution failed")]
Provider(ProviderError),
#[error("request was cancelled")]
@@ -123,7 +123,7 @@ impl UpstreamTransport {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderCallMetadata {
provider: ProviderKind,
upstream_model: UpstreamModelId,
upstream_model: Option<UpstreamModelId>,
resource: ProviderResource,
upstream_request_id: Option<OpaqueUpstreamValue>,
transport: UpstreamTransport,
@@ -140,7 +140,23 @@ impl ProviderCallMetadata {
) -> Self {
Self {
provider,
upstream_model,
upstream_model: Some(upstream_model),
resource,
upstream_request_id: None,
transport,
}
}
/// 创建一次不声明模型的 Provider 原生端点调用事实。
#[must_use]
pub const fn for_provider_endpoint(
provider: ProviderKind,
resource: ProviderResource,
transport: UpstreamTransport,
) -> Self {
Self {
provider,
upstream_model: None,
resource,
upstream_request_id: None,
transport,
@@ -160,10 +176,10 @@ impl ProviderCallMetadata {
&self.provider
}
/// 返回实际模型;必须与冻结 target 一致。
/// 返回实际模型;Provider 原生端点没有模型事实。
#[must_use]
pub const fn upstream_model(&self) -> &UpstreamModelId {
&self.upstream_model
pub const fn upstream_model(&self) -> Option<&UpstreamModelId> {
self.upstream_model.as_ref()
}
/// 返回匿名资源;账号调用返回 `None`。
@@ -199,7 +215,8 @@ impl ProviderCallMetadata {
/// 确认 metadata 没有替换请求计划中冻结的 Provider 候选。
#[must_use]
pub fn confirms(&self, candidate: &ProviderCandidate) -> bool {
candidate.provider() == &self.provider && candidate.upstream_model() == &self.upstream_model
candidate.provider() == &self.provider
&& candidate.upstream_model() == self.upstream_model.as_ref()
}
}
@@ -517,7 +534,7 @@ impl fmt::Debug for ProviderRequest {
.debug_struct("ProviderRequest")
.field("operation", &self.operation)
.field("provider", self.candidate.provider())
.field("upstream_model", self.candidate.upstream_model())
.field("upstream_model", &self.candidate.upstream_model())
.finish()
}
}
+15
View File
@@ -33,6 +33,9 @@ pub enum IdentifierError {
/// 规定前缀。
expected: &'static str,
},
/// 标识不满足该领域值对象的固定格式。
#[error("identifier has an invalid format")]
InvalidFormat,
}
/// Operation 构造或校验失败。
@@ -67,6 +70,12 @@ pub enum RoutingError {
/// 配置 revision 必须为正数。
#[error("config revision must be greater than zero")]
InvalidRevision,
/// Restricted scope 必须保留至少一个持久化分组 binding。
#[error("account routing scope is invalid")]
InvalidAccountScope,
/// Key 的账号范围中当前没有任何可参与路由的账号。
#[error("account routing scope is empty")]
EmptyAccountScope,
/// 快照中存在重复实体。
#[error("duplicate {entity} `{id}`")]
DuplicateEntity {
@@ -89,6 +98,12 @@ pub enum RoutingError {
/// 客户端提交的模型名称。
model: String,
},
/// 固定 Provider 的原生端点当前不可执行。
#[error("provider endpoint `{provider}` is unavailable")]
NoCapableProviderEndpoint {
/// adapter 已绑定的 Provider。
provider: String,
},
}
/// 调用方策略不满足约束或拒绝请求。
+55 -4
View File
@@ -16,7 +16,7 @@ use crate::operation::ProviderSessionState;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResponseMeta {
response_id: String,
model: String,
model: Option<String>,
finish_reason: Option<FinishReason>,
}
@@ -26,7 +26,17 @@ impl ResponseMeta {
pub fn new(response_id: impl Into<String>, model: impl Into<String>) -> Self {
Self {
response_id: response_id.into(),
model: model.into(),
model: Some(model.into()),
finish_reason: None,
}
}
/// 创建不声明模型的 Provider 原生端点响应元数据。
#[must_use]
pub fn for_provider_endpoint(response_id: impl Into<String>) -> Self {
Self {
response_id: response_id.into(),
model: None,
finish_reason: None,
}
}
@@ -46,8 +56,8 @@ impl ResponseMeta {
/// 返回对客户端公开的模型名。
#[must_use]
pub fn model(&self) -> &str {
&self.model
pub fn model(&self) -> Option<&str> {
self.model.as_deref()
}
/// 返回规范化终止原因。
@@ -190,6 +200,7 @@ pub struct ProtocolWireEvent {
data: Value,
has_json_data: bool,
raw_sse_frame: Option<Bytes>,
raw_json_body: Option<Bytes>,
sse_id: Option<String>,
sse_retry: Option<u64>,
}
@@ -228,6 +239,7 @@ impl ProtocolWireEvent {
data,
has_json_data: true,
raw_sse_frame: None,
raw_json_body: None,
sse_id,
sse_retry,
})
@@ -274,6 +286,32 @@ impl ProtocolWireEvent {
data: Value::Null,
has_json_data: false,
raw_sse_frame: Some(raw_sse_frame),
raw_json_body: None,
sse_id: None,
sse_retry: None,
})
}
/// 创建未经改写的完整 JSON 响应正文。
///
/// 该载荷用于非流式协议端点;Core 不解析或重编码其中的值。
///
/// # Errors
///
/// 协议名不满足内部路由标识约束时返回错误。
pub fn raw_json(
protocol: impl Into<String>,
raw_json_body: Bytes,
) -> Result<Self, IdentifierError> {
let protocol = protocol.into();
validate_text(&protocol, 64, true, None)?;
Ok(Self {
protocol,
event_type: None,
data: Value::Null,
has_json_data: false,
raw_sse_frame: None,
raw_json_body: Some(raw_json_body),
sse_id: None,
sse_retry: None,
})
@@ -321,6 +359,18 @@ impl ProtocolWireEvent {
self.raw_sse_frame.as_ref()
}
/// 返回可直接写给 JSON 客户端的未经改写响应正文。
#[must_use]
pub const fn raw_json_body(&self) -> Option<&Bytes> {
self.raw_json_body.as_ref()
}
/// 拆出未经改写的完整 JSON 响应正文。
#[must_use]
pub fn into_raw_json_body(self) -> Option<Bytes> {
self.raw_json_body
}
/// 拆出协议原生 JSON 数据。
#[must_use]
pub fn into_data(self) -> Value {
@@ -336,6 +386,7 @@ impl fmt::Debug for ProtocolWireEvent {
.field("event_type", &self.event_type)
.field("has_json_data", &self.has_json_data)
.field("has_raw_sse_frame", &self.raw_sse_frame.is_some())
.field("has_raw_json_body", &self.raw_json_body.is_some())
.field("has_sse_id", &self.sse_id.is_some())
.field("sse_retry", &self.sse_retry)
.field("data", &"<not included in Debug>")
@@ -1,13 +1,14 @@
//! 协议无关的业务 operation。
//!
//! 这里只保留网关需要解释、路由或结算的稳定语义。协议 adapter 无法写入
//! wire body 的连接级事实随 [`ProtocolPayload`] 不透明传递,由对应 Provider
//! 自己解释。
//! wire body 与连接级事实随 [`ProtocolPayload`] 或 [`RawJsonPayload`] 不透明传递,
//! 由对应 Provider 自己解释。
use std::collections::BTreeSet;
use std::fmt;
use std::sync::Arc;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
@@ -239,6 +240,72 @@ impl fmt::Debug for ProtocolPayload {
}
}
/// 客户端协议交给 Provider 的未经重编码 JSON 正文。
///
/// 协议 adapter 只确定路由端点;Core 保留原始字节与非 wire 上下文,
/// 不验证、解析或改写正文。
#[derive(Clone, PartialEq, Eq)]
pub struct RawJsonPayload {
protocol: String,
body: Bytes,
context: Map<String, Value>,
}
impl RawJsonPayload {
/// 创建未经重编码的协议 JSON 正文。
///
/// # Errors
///
/// 协议名称为空、过长或含控制字符时返回错误。
pub fn new(protocol: impl Into<String>, body: Bytes) -> Result<Self, OperationError> {
let protocol = protocol.into();
validate_text(&protocol, 64, true, None).map_err(|_| OperationError::EmptyField {
field: "raw_json_payload protocol",
})?;
Ok(Self {
protocol,
body,
context: Map::new(),
})
}
/// 返回协议名称。
#[must_use]
pub fn protocol(&self) -> &str {
&self.protocol
}
/// 返回 adapter 收到的原始 JSON 字节。
#[must_use]
pub const fn body(&self) -> &Bytes {
&self.body
}
/// 附着不写入 wire body 的协议连接上下文。
#[must_use]
pub fn with_context(mut self, context: Map<String, Value>) -> Self {
self.context = context;
self
}
/// 返回仅供对应 Provider 解释的非 wire 上下文。
#[must_use]
pub const fn context(&self) -> &Map<String, Value> {
&self.context
}
}
impl fmt::Debug for RawJsonPayload {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RawJsonPayload")
.field("protocol", &self.protocol)
.field("body", &"<not included in Debug>")
.field("context", &"<not included in Debug>")
.finish()
}
}
/// 通用生成请求。
#[derive(Clone, PartialEq)]
pub struct GenerateRequest {
@@ -501,32 +568,39 @@ impl fmt::Debug for RerankRequest {
}
}
/// 图像生成请求。
#[derive(Clone, PartialEq, Eq)]
/// 图像 API 的稳定端点语义。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ImageRequestKind {
/// 从文本生成图像。
Generation,
/// 使用输入图像执行编辑。
Edit,
}
/// 图像生成或编辑请求。
#[derive(Clone, PartialEq)]
pub struct ImageRequest {
prompt: String,
count: u32,
kind: ImageRequestKind,
payload: RawJsonPayload,
}
impl ImageRequest {
/// 创建图像生成请求。
///
/// # Errors
///
/// Prompt 为空时返回错误。
pub fn new(prompt: impl Into<String>) -> Result<Self, OperationError> {
let prompt = prompt.into();
if prompt.is_empty() {
return Err(OperationError::EmptyField { field: "prompt" });
}
Ok(Self { prompt, count: 1 })
/// 创建携带原始协议 JSON 正文的图像请求。
#[must_use]
pub const fn from_raw_json(kind: ImageRequestKind, payload: RawJsonPayload) -> Self {
Self { kind, payload }
}
/// 设置图像数量。
/// 返回图像 API 端点语义。
#[must_use]
pub const fn with_count(mut self, count: u32) -> Self {
self.count = count;
self
pub const fn kind(&self) -> ImageRequestKind {
self.kind
}
/// 返回未经重编码的协议 JSON 正文。
#[must_use]
pub const fn payload(&self) -> &RawJsonPayload {
&self.payload
}
}
@@ -534,8 +608,8 @@ impl fmt::Debug for ImageRequest {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ImageRequest")
.field("prompt", &"<not included in Debug>")
.field("count", &self.count)
.field("kind", &self.kind)
.field("payload", &"<not included in Debug>")
.finish()
}
}
@@ -649,7 +723,8 @@ impl Operation {
pub fn image_generation_requested(&self) -> bool {
match self {
Self::Generate(request) => request.image_generation_requested(),
Self::Embed(_) | Self::Rerank(_) | Self::GenerateImage(_) | Self::Speech(_) => false,
Self::GenerateImage(_) => true,
Self::Embed(_) | Self::Rerank(_) | Self::Speech(_) => false,
}
}
@@ -1,11 +1,12 @@
//! 下游 Client API Key 的准入策略。
//!
//! Client API Key 绑定一个 Provider 平台;模型名称不参与改写或 allowlist 判断。
//! Client API Key 冻结账号分组权限;模型名称不参与权限判断。
use std::fmt;
use std::sync::Arc;
use crate::error::{IdentifierError, PolicyError, validate_text};
use crate::routing::ProviderKind;
use crate::routing::FrozenAccountScope;
/// `client_api_keys.id` 的核心值对象。
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -88,7 +89,7 @@ impl RateLimits {
pub struct ClientPolicy {
key_id: ClientApiKeyId,
plaintext_key: PlaintextClientApiKey,
provider_kind: ProviderKind,
account_scope: Arc<FrozenAccountScope>,
enabled: bool,
limits: RateLimits,
}
@@ -98,14 +99,14 @@ impl ClientPolicy {
pub const fn new(
key_id: ClientApiKeyId,
plaintext_key: PlaintextClientApiKey,
provider_kind: ProviderKind,
account_scope: Arc<FrozenAccountScope>,
enabled: bool,
limits: RateLimits,
) -> Self {
Self {
key_id,
plaintext_key,
provider_kind,
account_scope,
enabled,
limits,
}
@@ -122,8 +123,8 @@ impl ClientPolicy {
}
#[must_use]
pub const fn provider_kind(&self) -> &ProviderKind {
&self.provider_kind
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
&self.account_scope
}
#[must_use]
@@ -9,10 +9,11 @@ use std::time::{Duration, SystemTime};
use futures::future::BoxFuture;
use crate::engine::credential::{
AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision,
AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, CredentialState,
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
};
use crate::error::{IdentifierError, validate_text};
use crate::policy::ClientApiKeyId;
use crate::routing::{ProviderKind, UpstreamModelId};
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
@@ -170,6 +171,7 @@ pub enum ProviderLeaseRequest {
pub trait ProviderLeasePort: Send + Sync {
fn load_state<'a>(
&'a self,
client_api_key_id: &'a ClientApiKeyId,
provider_kind: &'a ProviderKind,
accounts: &'a [ProviderAccountId],
) -> BoxFuture<'a, Result<ProviderSchedulingState, ProviderStoreError>>;
@@ -405,13 +407,91 @@ pub trait ProviderCatalogCachePort: Send + Sync {
) -> BoxFuture<'a, Result<Option<OpaqueProviderData>, ProviderStoreError>>;
}
/// Provider 从官方制品核验出的可重建请求画像。
///
/// Core 只用单调制品序号约束覆盖顺序;具体版本字段由对应 Provider 放在
/// `profile` 中解释。每个 Provider 在 Store 中只保留一份最新画像。
#[derive(Clone, PartialEq)]
pub struct ProviderArtifactProfile {
provider_kind: ProviderKind,
artifact_sequence: u64,
verified_at: SystemTime,
profile: OpaqueProviderData,
}
impl ProviderArtifactProfile {
#[must_use]
pub const fn new(
provider_kind: ProviderKind,
artifact_sequence: u64,
verified_at: SystemTime,
profile: OpaqueProviderData,
) -> Self {
Self {
provider_kind,
artifact_sequence,
verified_at,
profile,
}
}
#[must_use]
pub const fn provider_kind(&self) -> &ProviderKind {
&self.provider_kind
}
#[must_use]
pub const fn artifact_sequence(&self) -> u64 {
self.artifact_sequence
}
#[must_use]
pub const fn verified_at(&self) -> SystemTime {
self.verified_at
}
#[must_use]
pub const fn profile(&self) -> &OpaqueProviderData {
&self.profile
}
}
impl fmt::Debug for ProviderArtifactProfile {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ProviderArtifactProfile")
.field("provider_kind", &self.provider_kind)
.field("artifact_sequence", &self.artifact_sequence)
.field("verified_at", &self.verified_at)
.field("profile", &"[PROVIDER-OWNED]")
.finish()
}
}
pub trait ProviderArtifactProfileCachePort: Send + Sync {
/// 覆盖同一 Provider 的固定 cache key。
///
/// 返回 `false` 表示 Store 已持有更高的制品序号;相同序号但内容不同必须返回
/// [`ProviderStoreErrorKind::Conflict`],不能静默改写已核验画像。
fn replace_if_newer(
&self,
profile: ProviderArtifactProfile,
ttl: Duration,
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
fn read<'a>(
&'a self,
provider_kind: &'a ProviderKind,
) -> BoxFuture<'a, Result<Option<ProviderArtifactProfile>, ProviderStoreError>>;
}
/// Redis 中可重建的账号状态投影。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderCredentialState {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
}
@@ -421,14 +501,14 @@ impl ProviderCredentialState {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
) -> Self {
Self {
account_id,
credential_revision,
enabled,
availability,
credential_state,
observed_at,
}
}
@@ -449,8 +529,8 @@ impl ProviderCredentialState {
}
#[must_use]
pub const fn availability(&self) -> AccountAvailability {
self.availability
pub const fn credential_state(&self) -> CredentialState {
self.credential_state
}
#[must_use]
@@ -674,22 +754,20 @@ impl ProviderRefreshPolicy {
self.concurrency
}
/// 按配置的精确提前量计算刷新时刻。
pub fn next_attempt_at(
/// 判断 AT 是否已经进入当前配置的提前刷新窗口。
///
/// `margin` 是运行时策略,不投影为账号的持久化时间字段;已过期 AT 也需要
/// 尝试用 RT 恢复,因此视为到期。
#[must_use]
pub fn is_refresh_due(
self,
_account_id: &ProviderAccountId,
access_token_expires_at: SystemTime,
observed_at: SystemTime,
) -> Result<SystemTime, ProviderStoreError> {
let remaining = access_token_expires_at
.duration_since(observed_at)
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
if self.margin >= remaining {
return Ok(observed_at);
) -> bool {
match access_token_expires_at.duration_since(observed_at) {
Ok(remaining) => remaining <= self.margin,
Err(_) => true,
}
access_token_expires_at
.checked_sub(self.margin)
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
}
}
@@ -946,6 +1024,7 @@ pub struct ProviderStorePorts {
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
account_feedback: Arc<AccountFeedbackStats>,
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
credential_state: Arc<dyn ProviderCredentialStatePort>,
cooldowns: Arc<dyn ProviderCooldownPort>,
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
@@ -962,6 +1041,7 @@ impl ProviderStorePorts {
session_affinity: Arc<dyn ProviderSessionAffinityPort>,
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
credential_state: Arc<dyn ProviderCredentialStatePort>,
cooldowns: Arc<dyn ProviderCooldownPort>,
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
@@ -974,6 +1054,7 @@ impl ProviderStorePorts {
session_exclusions,
account_feedback: Arc::new(AccountFeedbackStats::default()),
catalog_cache,
artifact_profiles,
credential_state,
cooldowns,
runtime_policy,
@@ -1011,6 +1092,11 @@ impl ProviderStorePorts {
Arc::clone(&self.catalog_cache)
}
#[must_use]
pub fn artifact_profiles(&self) -> Arc<dyn ProviderArtifactProfileCachePort> {
Arc::clone(&self.artifact_profiles)
}
#[must_use]
pub fn credential_state(&self) -> Arc<dyn ProviderCredentialStatePort> {
Arc::clone(&self.credential_state)
+300 -13
View File
@@ -9,12 +9,293 @@ use std::fmt;
use std::num::{NonZeroU32, NonZeroU64};
use std::sync::Arc;
use crate::engine::credential::AccountSelectionPolicy;
use crate::engine::credential::{AccountSelectionPolicy, ProviderAccountId};
use crate::error::{IdentifierError, RoutingError, validate_text};
use crate::operation::{CapabilityRequirements, Feature, OperationKind};
const MAX_REQUEST_ATTEMPTS: u32 = 32;
/// `account_groups.id` 的核心值对象。
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct AccountGroupId(String);
impl AccountGroupId {
/// 校验并创建账号分组 ID。
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
let value = value.into();
let Some(suffix) = value.strip_prefix("grp_") else {
return Err(IdentifierError::MissingPrefix { expected: "grp_" });
};
if suffix.len() != 32
|| !suffix
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err(IdentifierError::InvalidFormat);
}
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for AccountGroupId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
/// 快照中一个账号的 Provider 与分组归属。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RuntimeAccount {
provider_kind: ProviderKind,
group_ids: Arc<BTreeSet<AccountGroupId>>,
}
impl RuntimeAccount {
#[must_use]
pub fn new(provider_kind: ProviderKind, group_ids: BTreeSet<AccountGroupId>) -> Self {
Self {
provider_kind,
group_ids: Arc::new(group_ids),
}
}
#[must_use]
pub const fn provider_kind(&self) -> &ProviderKind {
&self.provider_kind
}
#[must_use]
pub fn group_ids(&self) -> &BTreeSet<AccountGroupId> {
&self.group_ids
}
}
/// 全快照共享的账号、Provider 与分组反向索引。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RuntimeAccountDirectory {
accounts: BTreeMap<ProviderAccountId, RuntimeAccount>,
providers_with_accounts: BTreeSet<ProviderKind>,
providers_by_group: BTreeMap<AccountGroupId, BTreeSet<ProviderKind>>,
}
impl RuntimeAccountDirectory {
#[must_use]
pub fn new(accounts: BTreeMap<ProviderAccountId, RuntimeAccount>) -> Self {
let mut providers_with_accounts = BTreeSet::new();
let mut providers_by_group = BTreeMap::<AccountGroupId, BTreeSet<ProviderKind>>::new();
for account in accounts.values() {
providers_with_accounts.insert(account.provider_kind.clone());
for group_id in account.group_ids.iter() {
providers_by_group
.entry(group_id.clone())
.or_default()
.insert(account.provider_kind.clone());
}
}
Self {
accounts,
providers_with_accounts,
providers_by_group,
}
}
#[must_use]
pub fn account(&self, account_id: &ProviderAccountId) -> Option<&RuntimeAccount> {
self.accounts.get(account_id)
}
#[must_use]
pub fn providers_with_accounts(&self) -> &BTreeSet<ProviderKind> {
&self.providers_with_accounts
}
#[must_use]
pub fn providers_for_groups<'a>(
&self,
group_ids: impl IntoIterator<Item = &'a AccountGroupId>,
) -> BTreeSet<ProviderKind> {
group_ids
.into_iter()
.filter_map(|group_id| self.providers_by_group.get(group_id))
.flatten()
.cloned()
.collect()
}
}
/// 历史请求保存的账号范围种类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountRoutingScopeKind {
All,
Groups,
}
impl AccountRoutingScopeKind {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::All => "all",
Self::Groups => "groups",
}
}
}
/// 请求开始时冻结的分组 ID 与名称。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RoutingGroupSnapshot {
id: AccountGroupId,
name: String,
}
impl RoutingGroupSnapshot {
#[must_use]
pub fn new(id: AccountGroupId, name: String) -> Self {
Self { id, name }
}
#[must_use]
pub const fn id(&self) -> &AccountGroupId {
&self.id
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
}
/// 请求历史所需的完整、稳定账号范围快照。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountRoutingSnapshot {
kind: AccountRoutingScopeKind,
groups: Arc<[RoutingGroupSnapshot]>,
}
impl AccountRoutingSnapshot {
#[must_use]
pub fn all() -> Self {
Self {
kind: AccountRoutingScopeKind::All,
groups: Arc::from([]),
}
}
#[must_use]
pub fn groups(groups: Vec<RoutingGroupSnapshot>) -> Self {
Self {
kind: AccountRoutingScopeKind::Groups,
groups: Arc::from(groups),
}
}
#[must_use]
pub const fn kind(&self) -> AccountRoutingScopeKind {
self.kind
}
#[must_use]
pub fn groups_snapshot(&self) -> &[RoutingGroupSnapshot] {
&self.groups
}
}
/// Key 持久 binding 编译出的账号权限。
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ClientRoutingScope {
AllAccounts,
Restricted {
bound_groups: Arc<[RoutingGroupSnapshot]>,
enabled_group_ids: Arc<BTreeSet<AccountGroupId>>,
provider_kinds: Arc<BTreeSet<ProviderKind>>,
},
}
impl ClientRoutingScope {
#[must_use]
pub fn all_accounts() -> Self {
Self::AllAccounts
}
pub fn restricted(
bound_groups: Vec<RoutingGroupSnapshot>,
enabled_group_ids: BTreeSet<AccountGroupId>,
provider_kinds: BTreeSet<ProviderKind>,
) -> Result<Self, RoutingError> {
if bound_groups.is_empty() {
return Err(RoutingError::InvalidAccountScope);
}
Ok(Self::Restricted {
bound_groups: Arc::from(bound_groups),
enabled_group_ids: Arc::new(enabled_group_ids),
provider_kinds: Arc::new(provider_kinds),
})
}
}
/// 一次认证随 RuntimeSnapshot 冻结的账号目录与 Key 权限。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FrozenAccountScope {
directory: Arc<RuntimeAccountDirectory>,
client_scope: ClientRoutingScope,
}
impl FrozenAccountScope {
#[must_use]
pub const fn new(
directory: Arc<RuntimeAccountDirectory>,
client_scope: ClientRoutingScope,
) -> Self {
Self {
directory,
client_scope,
}
}
#[must_use]
pub fn allows(&self, account_id: &ProviderAccountId) -> bool {
let Some(account) = self.directory.account(account_id) else {
return false;
};
match &self.client_scope {
ClientRoutingScope::AllAccounts => true,
ClientRoutingScope::Restricted {
enabled_group_ids, ..
} => account
.group_ids()
.iter()
.any(|group_id| enabled_group_ids.contains(group_id)),
}
}
#[must_use]
pub fn provider_kinds(&self) -> &BTreeSet<ProviderKind> {
match &self.client_scope {
ClientRoutingScope::AllAccounts => self.directory.providers_with_accounts(),
ClientRoutingScope::Restricted { provider_kinds, .. } => provider_kinds,
}
}
#[must_use]
pub fn routing_snapshot(&self) -> AccountRoutingSnapshot {
match &self.client_scope {
ClientRoutingScope::AllAccounts => AccountRoutingSnapshot::all(),
ClientRoutingScope::Restricted { bound_groups, .. } => {
AccountRoutingSnapshot::groups(bound_groups.to_vec())
}
}
}
#[must_use]
pub const fn directory(&self) -> &Arc<RuntimeAccountDirectory> {
&self.directory
}
}
/// 编译进二进制的 Provider adapter slug。
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ProviderKind(String);
@@ -494,17 +775,18 @@ impl ProviderModel {
/// 本次请求选择 Provider 时使用的动态过滤事实。
#[derive(Debug, Clone, Default)]
pub struct RoutingContext {
/// 已认证 Client API Key 绑定的平台;模型名称不参与平台猜测。
pub provider_kind: Option<ProviderKind>,
/// 管理端 connection test 显式限制的 Provider;普通请求留空。
pub required_provider: Option<ProviderKind>,
pub blocked_providers: BTreeSet<ProviderKind>,
}
/// 已绑定 Provider 与真实上游模型的请求候选。
/// 已绑定 Provider 的请求候选;模型端点携带真实上游模型,原生端点不虚构模型。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderCandidate {
provider: ProviderKind,
upstream_model: UpstreamModelId,
upstream_model: Option<UpstreamModelId>,
emulated_features: BTreeSet<Feature>,
account_scope: Arc<FrozenAccountScope>,
}
impl ProviderCandidate {
@@ -514,14 +796,19 @@ impl ProviderCandidate {
}
#[must_use]
pub const fn upstream_model(&self) -> &UpstreamModelId {
&self.upstream_model
pub const fn upstream_model(&self) -> Option<&UpstreamModelId> {
self.upstream_model.as_ref()
}
#[must_use]
pub const fn emulated_features(&self) -> &BTreeSet<Feature> {
&self.emulated_features
}
#[must_use]
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
&self.account_scope
}
}
/// 一次请求冻结的 Provider 尝试顺序。
@@ -529,9 +816,9 @@ impl ProviderCandidate {
pub struct RoutingPlan {
config_revision: ConfigRevision,
account_selection_policy: AccountSelectionPolicy,
public_model: PublicModelId,
operation: OperationKind,
max_attempts: NonZeroU32,
account_scope: Arc<FrozenAccountScope>,
candidates: Arc<[ProviderCandidate]>,
}
@@ -546,11 +833,6 @@ impl RoutingPlan {
self.account_selection_policy
}
#[must_use]
pub const fn public_model(&self) -> &PublicModelId {
&self.public_model
}
#[must_use]
pub const fn operation(&self) -> OperationKind {
self.operation
@@ -561,6 +843,11 @@ impl RoutingPlan {
self.max_attempts
}
#[must_use]
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
&self.account_scope
}
#[must_use]
pub fn candidates(&self) -> &[ProviderCandidate] {
&self.candidates
@@ -11,7 +11,7 @@ use futures::{FutureExt as _, Stream, StreamExt as _, pin_mut, select_biased};
use futures_timer::Delay;
use crate::engine::CancellationToken;
use crate::engine::credential::{AccountSelectionPolicy, RotationStrategy};
use crate::engine::credential::{AccountSelectionPolicy, ProviderAccountId, RotationStrategy};
use crate::engine::provider::{ProviderCatalogGeneration, ProviderRegistry};
use crate::error::RoutingError;
use crate::health::{HealthProbe, HealthState};
@@ -24,8 +24,9 @@ use crate::task::{
};
use super::{
ConfigRevision, ModelCapabilities, ProviderCandidate, ProviderKind, ProviderModel,
PublicModelId, RoutingContext, RoutingPlan, UpstreamModelId,
AccountGroupId, ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities,
ProviderCandidate, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
RoutingGroupSnapshot, RoutingPlan, RuntimeAccount, RuntimeAccountDirectory, UpstreamModelId,
};
const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(5);
@@ -66,7 +67,7 @@ impl SnapshotSettingsFacts {
pub struct SnapshotClientPolicyFacts {
key_id: ClientApiKeyId,
plaintext_key: PlaintextClientApiKey,
provider_kind: String,
group_ids: Vec<AccountGroupId>,
limits: RateLimits,
}
@@ -75,18 +76,67 @@ impl SnapshotClientPolicyFacts {
pub fn new(
key_id: ClientApiKeyId,
plaintext_key: PlaintextClientApiKey,
provider_kind: impl Into<String>,
group_ids: Vec<AccountGroupId>,
limits: RateLimits,
) -> Self {
Self {
key_id,
plaintext_key,
provider_kind: provider_kind.into(),
group_ids,
limits,
}
}
}
/// Store 读取到的账号分组事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotAccountGroupFacts {
id: AccountGroupId,
name: String,
enabled: bool,
}
impl SnapshotAccountGroupFacts {
#[must_use]
pub fn new(id: AccountGroupId, name: String, enabled: bool) -> Self {
Self { id, name, enabled }
}
}
/// Store 读取到的账号及其固有 Provider 事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotProviderAccountFacts {
account_id: ProviderAccountId,
provider_kind: String,
}
impl SnapshotProviderAccountFacts {
#[must_use]
pub fn new(account_id: ProviderAccountId, provider_kind: impl Into<String>) -> Self {
Self {
account_id,
provider_kind: provider_kind.into(),
}
}
}
/// Store 读取到的一条分组成员关系。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotAccountGroupMemberFacts {
group_id: AccountGroupId,
account_id: ProviderAccountId,
}
impl SnapshotAccountGroupMemberFacts {
#[must_use]
pub const fn new(group_id: AccountGroupId, account_id: ProviderAccountId) -> Self {
Self {
group_id,
account_id,
}
}
}
/// 一次一致性读取产生的全部 RuntimeSnapshot 持久事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotFacts {
@@ -94,6 +144,9 @@ pub struct SnapshotFacts {
observed_current_revision: ConfigRevision,
settings: SnapshotSettingsFacts,
client_policies: Vec<SnapshotClientPolicyFacts>,
account_groups: Vec<SnapshotAccountGroupFacts>,
provider_accounts: Vec<SnapshotProviderAccountFacts>,
group_memberships: Vec<SnapshotAccountGroupMemberFacts>,
}
impl SnapshotFacts {
@@ -103,12 +156,18 @@ impl SnapshotFacts {
observed_current_revision: ConfigRevision,
settings: SnapshotSettingsFacts,
client_policies: Vec<SnapshotClientPolicyFacts>,
account_groups: Vec<SnapshotAccountGroupFacts>,
provider_accounts: Vec<SnapshotProviderAccountFacts>,
group_memberships: Vec<SnapshotAccountGroupMemberFacts>,
) -> Self {
Self {
config_revision,
observed_current_revision,
settings,
client_policies,
account_groups,
provider_accounts,
group_memberships,
}
}
@@ -223,13 +282,16 @@ async fn compile_runtime_snapshot(
providers: &ProviderRegistry,
) -> Result<RuntimeSnapshot, RuntimeSnapshotCompileError> {
let provider_kinds = providers.provider_kinds().cloned().collect::<Vec<_>>();
let registered_providers = provider_kinds.iter().cloned().collect::<BTreeSet<_>>();
// 实时目录只提供公开模型与能力提示;查询失败时保留 Provider 透传语义。
// 目录查询失败表示未知;查询成功后,即使为空,也必须与“已知缺少模型”区分。
let mut provider_models = Vec::new();
let mut known_provider_catalogs = BTreeSet::new();
for provider in &provider_kinds {
let Ok(models) = providers.query_model_capabilities(provider).await else {
continue;
};
known_provider_catalogs.insert(provider.clone());
provider_models.extend(models.into_iter().map(|model| {
let compiled = ProviderModel::new(
provider.clone(),
@@ -243,6 +305,56 @@ async fn compile_runtime_snapshot(
}));
}
let mut groups = BTreeMap::new();
for group in facts.account_groups {
if group.name.trim() != group.name
|| group.name.is_empty()
|| group.name.chars().count() > 100
|| group.name.chars().any(char::is_control)
|| groups.insert(group.id.clone(), group).is_some()
{
return Err(RuntimeSnapshotCompileError::InvalidData);
}
}
let mut account_groups = BTreeMap::<ProviderAccountId, BTreeSet<AccountGroupId>>::new();
let mut accounts = BTreeMap::new();
for account in facts.provider_accounts {
let provider_kind = ProviderKind::new(account.provider_kind)
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?;
if !registered_providers.contains(&provider_kind)
|| accounts
.insert(
account.account_id.clone(),
RuntimeAccount::new(provider_kind, BTreeSet::new()),
)
.is_some()
{
return Err(RuntimeSnapshotCompileError::InvalidData);
}
account_groups.insert(account.account_id, BTreeSet::new());
}
let mut memberships = BTreeSet::new();
for membership in facts.group_memberships {
if !groups.contains_key(&membership.group_id)
|| !accounts.contains_key(&membership.account_id)
|| !memberships.insert((membership.group_id.clone(), membership.account_id.clone()))
{
return Err(RuntimeSnapshotCompileError::InvalidData);
}
account_groups
.get_mut(&membership.account_id)
.ok_or(RuntimeSnapshotCompileError::InvalidData)?
.insert(membership.group_id);
}
for (account_id, group_ids) in account_groups {
let account = accounts
.get_mut(&account_id)
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
*account = RuntimeAccount::new(account.provider_kind().clone(), group_ids);
}
let account_directory = Arc::new(RuntimeAccountDirectory::new(accounts));
let model_mappings = facts.settings.model_mappings;
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
@@ -252,21 +364,48 @@ async fn compile_runtime_snapshot(
.ok_or(RuntimeSnapshotCompileError::InvalidData)?,
Duration::from_millis(facts.settings.request_interval_ms),
);
let client_policies = facts
.client_policies
.into_iter()
.map(|policy| {
let provider = ProviderKind::new(policy.provider_kind)
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?;
Ok(ClientPolicy::new(
policy.key_id,
policy.plaintext_key,
provider,
true,
policy.limits,
))
})
.collect::<Result<Vec<_>, RuntimeSnapshotCompileError>>()?;
let mut client_policies = Vec::with_capacity(facts.client_policies.len());
for policy in facts.client_policies {
let account_scope = if policy.group_ids.is_empty() {
FrozenAccountScope::new(
Arc::clone(&account_directory),
ClientRoutingScope::all_accounts(),
)
} else {
let mut seen = BTreeSet::new();
let mut bound_groups = Vec::with_capacity(policy.group_ids.len());
let mut enabled_group_ids = BTreeSet::new();
for group_id in policy.group_ids {
if !seen.insert(group_id.clone()) {
return Err(RuntimeSnapshotCompileError::InvalidData);
}
let group = groups
.get(&group_id)
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
bound_groups.push(RoutingGroupSnapshot::new(
group.id.clone(),
group.name.clone(),
));
if group.enabled {
enabled_group_ids.insert(group_id);
}
}
bound_groups.sort_by(|left, right| left.id().cmp(right.id()));
let provider_kinds = account_directory.providers_for_groups(&enabled_group_ids);
FrozenAccountScope::new(
Arc::clone(&account_directory),
ClientRoutingScope::restricted(bound_groups, enabled_group_ids, provider_kinds)
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?,
)
};
client_policies.push(ClientPolicy::new(
policy.key_id,
policy.plaintext_key,
Arc::new(account_scope),
true,
policy.limits,
));
}
RuntimeSnapshot::new(
facts.config_revision,
@@ -276,7 +415,12 @@ async fn compile_runtime_snapshot(
client_policies,
)
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)
.map(|snapshot| snapshot.with_model_mappings(model_mappings))
.map(|snapshot| {
snapshot
.with_model_mappings(model_mappings)
.with_account_directory(account_directory)
.with_known_provider_catalogs(known_provider_catalogs)
})
}
/// 数据面使用的不可变配置快照。
@@ -290,6 +434,8 @@ pub struct RuntimeSnapshot {
Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>>,
model_mappings: Arc<BTreeMap<String, String>>,
provider_catalog_generations: Arc<BTreeMap<ProviderKind, ProviderCatalogGeneration>>,
known_provider_catalogs: Arc<BTreeSet<ProviderKind>>,
account_directory: Arc<RuntimeAccountDirectory>,
client_policies: Arc<BTreeMap<ClientApiKeyId, ClientPolicy>>,
}
@@ -312,6 +458,7 @@ impl RuntimeSnapshot {
}
}
let mut known_provider_catalogs = BTreeSet::new();
let mut model_map =
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>::new();
let mut presentation_map =
@@ -323,6 +470,7 @@ impl RuntimeSnapshot {
capabilities,
presentation,
} = model;
known_provider_catalogs.insert(provider.clone());
if !provider_set.contains(&provider) {
return Err(RoutingError::NotFound {
entity: "provider",
@@ -367,6 +515,8 @@ impl RuntimeSnapshot {
provider_model_presentations: Arc::new(presentation_map),
model_mappings: Arc::new(BTreeMap::new()),
provider_catalog_generations: Arc::new(BTreeMap::new()),
known_provider_catalogs: Arc::new(known_provider_catalogs),
account_directory: Arc::new(RuntimeAccountDirectory::default()),
client_policies: Arc::new(client_policy_map),
})
}
@@ -377,6 +527,26 @@ impl RuntimeSnapshot {
self
}
#[must_use]
pub fn with_account_directory(mut self, directory: Arc<RuntimeAccountDirectory>) -> Self {
self.account_directory = directory;
self
}
#[must_use]
fn with_known_provider_catalogs(mut self, providers: BTreeSet<ProviderKind>) -> Self {
self.known_provider_catalogs = Arc::new(providers);
self
}
#[must_use]
pub fn all_account_scope(&self) -> Arc<FrozenAccountScope> {
Arc::new(FrozenAccountScope::new(
Arc::clone(&self.account_directory),
ClientRoutingScope::all_accounts(),
))
}
#[must_use]
fn with_provider_catalog_generations(
mut self,
@@ -459,8 +629,39 @@ impl RuntimeSnapshot {
.collect()
}
/// 已取得目录时按目录与全局别名判断模型可用性;目录暂不可用时沿用旧版
/// fail-open 语义,把具体 wire 可接受性留给上游。
/// 合并冻结账号范围内实际存在 Provider 的公开模型。
#[must_use]
pub fn public_models_for_scope(&self, scope: &FrozenAccountScope) -> Vec<PublicModelId> {
scope
.provider_kinds()
.iter()
.flat_map(|provider| self.public_models_for_provider(provider))
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
/// 合并冻结账号范围内实际存在 Provider 的公开模型画像。
#[must_use]
pub fn public_model_profiles_for_scope(
&self,
scope: &FrozenAccountScope,
) -> Vec<super::PublicModelProfile> {
let mut profiles = BTreeMap::new();
for provider in scope.provider_kinds() {
for profile in self.public_model_profiles_for_provider(provider) {
profiles
.entry(profile.model().clone())
.or_insert_with(|| profile.presentation().clone());
}
}
profiles
.into_iter()
.map(|(model, presentation)| super::PublicModelProfile::new(model, presentation))
.collect()
}
/// 已取得目录时以映射后的上游模型为准;目录不可用时由 Provider 在发送前判定。
#[must_use]
pub fn contains_public_model_for_provider(
&self,
@@ -470,17 +671,25 @@ impl RuntimeSnapshot {
if !self.providers.contains(provider) {
return false;
}
if self.model_mappings.contains_key(public_model.as_str()) {
return true;
}
let upstream_model = self.mapped_model(public_model.as_str());
match self.provider_models.get(provider) {
Some(models) if !models.is_empty() => models
.keys()
.any(|model| model.as_str() == public_model.as_str()),
Some(_) | None => true,
Some(models) => models.keys().any(|model| model.as_str() == upstream_model),
None => !self.known_provider_catalogs.contains(provider),
}
}
#[must_use]
pub fn contains_public_model_for_scope(
&self,
public_model: &PublicModelId,
scope: &FrozenAccountScope,
) -> bool {
scope
.provider_kinds()
.iter()
.any(|provider| self.contains_public_model_for_provider(public_model, provider))
}
#[must_use]
pub fn mapped_model(&self, requested: &str) -> String {
let original = requested;
@@ -506,14 +715,26 @@ impl RuntimeSnapshot {
&self,
public_model: &PublicModelId,
operation: &Operation,
account_scope: Arc<FrozenAccountScope>,
context: &RoutingContext,
) -> Result<RoutingPlan, RoutingError> {
let requirements = operation.capability_requirements();
let mut candidates = Vec::new();
for provider in self.providers.iter() {
if context.required_provider.is_none() && account_scope.provider_kinds().is_empty() {
return Err(RoutingError::EmptyAccountScope);
}
let providers = context.required_provider.as_ref().map_or_else(
|| account_scope.provider_kinds().clone(),
|provider| BTreeSet::from([provider.clone()]),
);
for provider in &providers {
if !self.providers.contains(provider) {
continue;
}
if context
.provider_kind
.required_provider
.as_ref()
.is_some_and(|expected| expected != provider)
|| context.blocked_providers.contains(provider)
@@ -539,12 +760,14 @@ impl RuntimeSnapshot {
};
emulated
}
None if self.known_provider_catalogs.contains(provider) => continue,
None => BTreeSet::new(),
};
candidates.push(ProviderCandidate {
provider: provider.clone(),
upstream_model,
upstream_model: Some(upstream_model),
emulated_features,
account_scope: Arc::clone(&account_scope),
});
}
@@ -557,13 +780,54 @@ impl RuntimeSnapshot {
Ok(RoutingPlan {
config_revision: self.revision,
account_selection_policy: self.account_selection_policy,
public_model: public_model.clone(),
operation: operation.kind(),
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
.expect("constant request attempt limit is non-zero"),
account_scope,
candidates: Arc::from(candidates),
})
}
/// 为 Provider 自有、且不属于文本模型目录的端点冻结请求计划。
///
/// 端点 adapter 已经确定 Provider,因此这里只执行账号范围、circuit 和注册
/// 状态检查;不会读取业务正文、构造模型或查询 Provider 文本模型目录。
pub fn plan_provider_endpoint(
&self,
provider: &ProviderKind,
operation: &Operation,
account_scope: Arc<FrozenAccountScope>,
context: &RoutingContext,
) -> Result<RoutingPlan, RoutingError> {
let allowed = !account_scope.provider_kinds().is_empty()
&& account_scope.provider_kinds().contains(provider)
&& self.providers.contains(provider)
&& context
.required_provider
.as_ref()
.is_none_or(|required| required == provider)
&& !context.blocked_providers.contains(provider);
if !allowed {
return Err(RoutingError::NoCapableProviderEndpoint {
provider: provider.as_str().to_owned(),
});
}
let candidate = ProviderCandidate {
provider: provider.clone(),
upstream_model: None,
emulated_features: BTreeSet::new(),
account_scope: Arc::clone(&account_scope),
};
Ok(RoutingPlan {
config_revision: self.revision,
account_selection_policy: self.account_selection_policy,
operation: operation.kind(),
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
.expect("constant request attempt limit is non-zero"),
account_scope,
candidates: Arc::from([candidate]),
})
}
}
/// 请求级冻结失败;此状态必须 fail closed。
@@ -2,12 +2,14 @@ use gateway_core::engine::continuation::{
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
};
use gateway_core::engine::credential::ProviderAccountId;
use gateway_core::policy::ClientApiKeyId;
use gateway_core::routing::ProviderKind;
fn pin() -> NativeContinuationPin {
NativeContinuationPin::new(
PreviousResponseId::new("response-private"),
PreviousResponseId::new("upstream-private"),
ClientApiKeyId::new("key_client").expect("valid client key"),
ProviderKind::new("openai").expect("valid provider"),
ProviderAccountId::new("acct_codex").expect("valid account"),
)
@@ -43,3 +45,8 @@ fn native_pin_should_reject_different_account() {
&ProviderAccountId::new("acct_other").expect("valid account"),
));
}
#[test]
fn native_pin_should_reject_different_client_api_key() {
assert!(!pin().matches_client(&ClientApiKeyId::new("key_other").expect("valid client key")));
}
@@ -43,8 +43,9 @@ use gateway_core::operation::{
};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::routing::{
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
RoutingPlan, RuntimeSnapshot, UpstreamModelId,
AccountRoutingSnapshot, ClientRoutingScope, ConfigRevision, FrozenAccountScope,
ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext, RoutingPlan,
RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot, UpstreamModelId,
};
use serde_json::{Map, Value, json};
@@ -316,7 +317,10 @@ impl Provider for ScriptedProvider {
let candidate = request.candidate();
let metadata = ProviderCallMetadata::new(
candidate.provider().clone(),
candidate.upstream_model().clone(),
candidate
.upstream_model()
.cloned()
.expect("model route candidate"),
ProviderResource::Account {
id: ProviderAccountId::new(account_id).expect("account id"),
revision: CredentialRevision::new(1).expect("revision"),
@@ -336,7 +340,10 @@ impl Provider for ScriptedProvider {
let candidate = request.candidate();
let metadata = ProviderCallMetadata::new(
candidate.provider().clone(),
candidate.upstream_model().clone(),
candidate
.upstream_model()
.cloned()
.expect("model route candidate"),
ProviderResource::Account {
id: ProviderAccountId::new(account_id).expect("account id"),
revision: CredentialRevision::new(1).expect("revision"),
@@ -356,7 +363,10 @@ impl Provider for ScriptedProvider {
let candidate = request.candidate();
let metadata = ProviderCallMetadata::new(
candidate.provider().clone(),
candidate.upstream_model().clone(),
candidate
.upstream_model()
.cloned()
.expect("model route candidate"),
ProviderResource::Account {
id: ProviderAccountId::new(account_id).expect("account id"),
revision: CredentialRevision::new(1).expect("revision"),
@@ -492,6 +502,31 @@ fn bare_atomic_response_failed(response_id: &str) -> ProviderError {
.with_atomic_client_events(vec![failed])
}
fn atomic_history_unavailable(response_id: &str) -> ProviderError {
let failed = ProviderEvent::wire(
ProtocolWireEvent::json(
"openai",
Some("response.failed".to_owned()),
json!({
"type": "response.failed",
"response": {
"id": response_id,
"status": "failed",
"error": {
"code": "previous_response_not_found",
"message": "Previous response was not found. Retrying the full request."
}
}
}),
)
.expect("history unavailable wire"),
);
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
.with_status(400)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_atomic_client_events(vec![failed])
}
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
let mut usage = Usage::new();
usage.input_tokens = Some(12);
@@ -513,17 +548,52 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
}
fn plan(operation: &Operation) -> RoutingPlan {
let provider = ProviderKind::new("openai").expect("provider");
let public_model = PublicModelId::new("gpt-5").expect("public model");
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
.with_upstream_feature_validation();
RuntimeSnapshot::new(
ConfigRevision::new(1).expect("config revision"),
plan_with_policy(
operation,
AccountSelectionPolicy::new(
RotationStrategy::Smart,
NonZeroU32::new(2).expect("account concurrency"),
Duration::from_millis(50),
),
)
}
fn plan_with_policy(
operation: &Operation,
account_selection_policy: AccountSelectionPolicy,
) -> RoutingPlan {
let provider = ProviderKind::new("openai").expect("provider");
let public_model = PublicModelId::new("gpt-5").expect("public model");
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
.with_upstream_feature_validation();
let directory = Arc::new(RuntimeAccountDirectory::new(
[
"acct_failed",
"acct_first",
"acct_image",
"acct_observation_mismatch",
"acct_observed",
"acct_one",
"acct_only",
"acct_other",
"acct_required",
"acct_second",
"acct_tool",
"acct_two",
"acct_wrong",
]
.into_iter()
.map(|id| {
(
ProviderAccountId::new(id).expect("account"),
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
)
})
.collect(),
));
let snapshot = RuntimeSnapshot::new(
ConfigRevision::new(1).expect("config revision"),
account_selection_policy,
vec![provider.clone()],
vec![ProviderModel::new(
provider.clone(),
@@ -533,15 +603,21 @@ fn plan(operation: &Operation) -> RoutingPlan {
Vec::new(),
)
.expect("snapshot")
.plan(
&public_model,
operation,
&RoutingContext {
provider_kind: Some(provider),
..RoutingContext::default()
},
)
.expect("routing plan")
.with_account_directory(Arc::clone(&directory));
snapshot
.plan(
&public_model,
operation,
Arc::new(FrozenAccountScope::new(
directory,
ClientRoutingScope::all_accounts(),
)),
&RoutingContext {
required_provider: Some(provider),
..RoutingContext::default()
},
)
.expect("routing plan")
}
fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest {
@@ -551,11 +627,12 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
client_api_key_id: Some(client_key.clone()),
client_api_key_ref: client_key,
config_revision: ConfigRevision::new(1).expect("config revision"),
routing: AccountRoutingSnapshot::all(),
protocol: "openai".to_owned(),
operation: operation.kind(),
endpoint: "responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model: PublicModelId::new("gpt-5").expect("model"),
requested_model: Some(PublicModelId::new("gpt-5").expect("model")),
client_ip: Some("127.0.0.1".parse().expect("client IP")),
user_agent: Some("gateway-core-test".to_owned()),
reasoning_effort: Some("medium".to_owned()),
@@ -1344,6 +1421,7 @@ fn authenticated_native_continuation_reaches_every_attempt_context() {
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-secret-id"),
PreviousResponseId::new("provider-native-id"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
ProviderAccountId::new("acct_one").expect("account"),
);
@@ -1401,6 +1479,7 @@ fn streaming_native_continuation_delivers_started_event_before_later_output() {
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
ProviderAccountId::new("acct_one").expect("account"),
);
@@ -1436,7 +1515,7 @@ fn streaming_native_continuation_delivers_started_event_before_later_output() {
}
#[test]
fn native_continuation_replays_owner_before_safely_switching_account() {
fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery() {
let Operation::Generate(generate) = generate_operation() else {
panic!("generate operation");
};
@@ -1477,6 +1556,7 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-secret-id"),
PreviousResponseId::new("provider-native-id"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
ProviderAccountId::new("acct_one").expect("account"),
);
@@ -1490,7 +1570,7 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
CancellationToken::new(),
))
.expect("start execution");
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
block_on(session.collect_uncommitted()).expect("provider-defined recovery succeeds");
block_on(session.commit_downstream(Some(200))).expect("commit response");
let contexts = provider.contexts.lock().expect("contexts lock");
@@ -1514,6 +1594,141 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
);
}
#[test]
fn unavailable_native_continuation_uses_provider_recovery_with_the_configured_account_policy() {
let Operation::Generate(generate) = generate_operation() else {
panic!("generate operation");
};
let operation = Operation::Generate(generate.with_provider_session_state(
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
));
let policy = AccountSelectionPolicy::new(
RotationStrategy::RoundRobin,
NonZeroU32::new(2).expect("account concurrency"),
Duration::from_millis(50),
);
let route_plan = plan_with_policy(&operation, policy);
let (coordinator, store, provider) = coordinator(vec![
Script::Error(ProviderError::new(
ProviderErrorKind::NoEligibleAccount,
UpstreamSendState::NotSent,
)),
Script::Stream {
account_id: "acct_two",
items: complete_stream(None),
},
]);
let original = ProviderAccountId::new("acct_one").expect("account");
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
original.clone(),
);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
Some(ContinuationBinding::Pinned(continuation)),
CancellationToken::new(),
))
.expect("start execution");
block_on(session.collect_uncommitted()).expect("provider-defined recovery succeeds");
block_on(session.commit_downstream(Some(200))).expect("commit response");
let contexts = provider.contexts.lock().expect("contexts lock");
assert_eq!(contexts.len(), 2);
assert_eq!(
contexts[0].continuation_attempt(),
ContinuationAttempt::Native
);
assert_eq!(
contexts[1].continuation_attempt(),
ContinuationAttempt::ReplayAny
);
assert_eq!(
contexts[1].account_selection_policy().strategy(),
RotationStrategy::RoundRobin
);
assert!(contexts[1].excluded_accounts().contains(&original));
let state = store.state.lock().expect("store lock");
assert_eq!(state.attempts.len(), 1);
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
}
#[test]
fn continuation_history_failure_without_replay_proof_is_delivered_without_hidden_retry() {
let Operation::Generate(generate) = generate_operation() else {
panic!("generate operation");
};
let operation = Operation::Generate(generate.with_provider_session_state(
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
));
let route_plan = plan(&operation);
let (coordinator, _, provider) = coordinator(vec![
Script::ObservedStream {
account_id: "acct_one",
items: vec![Err(atomic_history_unavailable("response-history-missing"))],
},
Script::Stream {
account_id: "acct_two",
items: complete_stream(None),
},
]);
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
ProviderAccountId::new("acct_one").expect("account"),
);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
Some(ContinuationBinding::Pinned(continuation)),
CancellationToken::new(),
))
.expect("start execution");
let terminal = block_on(session.next_event())
.expect("history failure remains deliverable")
.expect("terminal history failure batch");
assert_eq!(
terminal.commit_requirement(),
CommitRequirement::CommitBeforeDelivery
);
let events = terminal.into_provider_events();
let failure = events
.iter()
.filter_map(ProviderEvent::wire_event)
.find(|wire| wire.event_type() == Some("response.failed"))
.expect("raw response.failed event");
assert_eq!(
failure
.data()
.pointer("/response/error/code")
.and_then(Value::as_str),
Some("previous_response_not_found")
);
block_on(session.commit_downstream(Some(200))).expect("commit terminal history failure");
assert!(matches!(
block_on(session.next_event()),
Err(EngineError::Provider(_))
));
let contexts = provider.contexts.lock().expect("contexts lock");
assert_eq!(contexts.len(), 1);
assert_eq!(
contexts[0].continuation_attempt(),
ContinuationAttempt::Native
);
}
#[test]
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
let operation = operation(RetrySafety::Idempotent);
@@ -2424,6 +2639,7 @@ fn native_continuation_explicit_429_is_not_retried() {
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-secret-id"),
PreviousResponseId::new("provider-native-id"),
ClientApiKeyId::new("key_client_1").expect("client key"),
ProviderKind::new("openai").expect("provider"),
ProviderAccountId::new("acct_first").expect("account"),
);
@@ -1,30 +1,39 @@
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use serde_json::{Map, Value};
use gateway_core::engine::credential::{
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
AccountAttemptFeedback, AccountCandidate, AccountConcurrencyLimit, AccountEligibilityPolicy,
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
RotationStrategy,
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, AccountStatus, AccountWeight,
CredentialCasUpdate, CredentialRevision, CredentialState, OpaqueProviderData,
PlaintextCredential, PreferredAccountSelection, ProviderAccount, ProviderAccountId,
ProviderAccountIdentity, ProviderAccountUpdate, QuotaEvidence, QuotaState, RotationStrategy,
};
use gateway_core::routing::{
ClientRoutingScope, FrozenAccountScope, ProviderKind, RuntimeAccount, RuntimeAccountDirectory,
};
use gateway_core::routing::ProviderKind;
fn account(id: &str) -> ProviderAccount {
ProviderAccount::new(
ProviderAccountId::new(id).expect("valid account"),
ProviderKind::new("openai").expect("valid provider"),
id.to_owned(),
format!("user-{id}"),
Some(format!("user-{id}")),
"oauth".to_owned(),
CredentialRevision::new(1).expect("valid revision"),
Some(SystemTime::now() + Duration::from_secs(3600)),
)
.with_runtime_state(true, AccountAvailability::Ready)
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
}
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
@@ -35,13 +44,50 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: remaining,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
}
}
fn candidate_with_concurrency(id: &str, in_flight: u32, concurrency: u32) -> AccountCandidate {
let mut candidate = candidate(id, in_flight, Some(100));
candidate.account = candidate.account.with_scheduling(
Some(AccountConcurrencyLimit::new(concurrency).expect("concurrency override")),
AccountWeight::DEFAULT,
);
candidate
}
fn weighted_candidate(id: &str, weight: u16, in_flight: u32) -> AccountCandidate {
let mut candidate = candidate(id, in_flight, None);
candidate.account = candidate.account.with_scheduling(
None,
AccountWeight::new(weight).expect("valid account weight"),
);
candidate
}
#[test]
fn account_scheduling_should_default_and_resolve_concurrency_override() {
let default = NonZeroU32::new(12).expect("default concurrency");
let account = account("acct_scheduling_default");
assert_eq!(account.concurrency_limit(), None);
assert_eq!(account.weight(), AccountWeight::DEFAULT);
assert_eq!(account.effective_concurrency(default), default);
let limit = AccountConcurrencyLimit::new(7).expect("concurrency override");
let weight = AccountWeight::new(80).expect("weight");
let account = account.with_scheduling(Some(limit), weight);
assert_eq!(account.concurrency_limit(), Some(limit));
assert_eq!(account.weight(), weight);
assert_eq!(account.effective_concurrency(default).get(), 7);
assert!(AccountConcurrencyLimit::new(0).is_none());
assert!(AccountWeight::new(0).is_none());
assert!(AccountWeight::new(101).is_none());
}
fn context(strategy: RotationStrategy) -> AccountSelectionContext {
AccountSelectionContext {
policy: AccountSelectionPolicy::new(
@@ -53,7 +99,8 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
excluded_accounts: BTreeSet::new(),
preferred_account: None,
round_robin_cursor: 0,
availability: AccountAvailabilityPolicy::Enforce,
eligibility: AccountEligibilityPolicy::Enforce,
account_scope: None,
}
}
@@ -101,43 +148,99 @@ fn provider_account_identity_debug_should_redact_upstream_ids() {
}
#[test]
fn availability_should_round_trip_all_database_values() {
fn account_fact_values_should_round_trip() {
assert_eq!(
AccountAvailability::parse(AccountAvailability::QuotaExhausted.as_str()),
Some(AccountAvailability::QuotaExhausted)
CredentialState::parse(CredentialState::Banned.as_str()),
Some(CredentialState::Banned)
);
assert_eq!(
QuotaEvidence::parse("provider_denied"),
Some(QuotaEvidence::ProviderDenied)
);
}
#[test]
fn diagnostic_selection_bypasses_all_local_account_availability() {
fn exhausted_quota_refresh_is_due_at_reset_or_provider_fallback() {
let now = SystemTime::now();
let observed_at = now - Duration::from_secs(60);
let before_reset = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
observed_at,
Some(now + Duration::from_secs(1)),
);
let after_reset = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
observed_at,
Some(now - Duration::from_secs(1)),
);
let without_reset = QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None);
assert!(!before_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(after_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(without_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(!without_reset.exhaustion_refresh_due(now, Duration::from_secs(120)));
}
#[test]
fn inconclusive_quota_observation_cannot_erase_confirmed_access() {
let now = SystemTime::now();
let exhausted = QuotaState::exhausted(QuotaEvidence::ProviderDenied, now, None);
assert_eq!(
exhausted
.merge_observation(QuotaState::observed_unknown(now + Duration::from_secs(1)))
.access(),
gateway_core::engine::credential::QuotaAccessState::Exhausted
);
assert_eq!(
exhausted
.merge_observation(QuotaState::allowed(now + Duration::from_secs(1)))
.access(),
gateway_core::engine::credential::QuotaAccessState::Allowed
);
}
#[test]
fn diagnostic_selection_bypasses_all_local_account_eligibility() {
let observed_at = SystemTime::now();
let exhausted = AccountCandidate {
account: account("acct_exhausted")
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
account: account("acct_exhausted").with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
None,
None,
),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
};
let disabled = AccountCandidate {
account: account("acct_disabled")
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
account: account("acct_disabled").with_account_facts(
false,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
None,
None,
),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
};
let mut context = context(RotationStrategy::Sticky);
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
context.eligibility = AccountEligibilityPolicy::BypassForDiagnostic;
assert_eq!(
AccountSelector
@@ -154,10 +257,91 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
}
#[test]
fn disabled_account_should_not_be_schedulable() {
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
fn selector_should_reject_a_preferred_account_outside_the_frozen_client_scope() {
let allowed = candidate("acct_allowed", 0, None);
let outside = candidate("acct_outside", 0, None);
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
allowed.account.id().clone(),
RuntimeAccount::new(
ProviderKind::new("openai").expect("provider"),
BTreeSet::new(),
),
)])));
let mut context = context(RotationStrategy::Sticky);
context.preferred_account = Some(outside.account.id().clone());
context.account_scope = Some(Arc::new(FrozenAccountScope::new(
directory,
ClientRoutingScope::all_accounts(),
)));
assert!(!account.is_schedulable(SystemTime::now()));
let candidates = [outside, allowed];
let selection = AccountSelector
.select(&candidates, &context)
.expect("allowed account should remain eligible");
assert_eq!(selection.candidate().account.id().as_str(), "acct_allowed");
assert_eq!(
selection.preferred(),
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::OutsideClientScope)
);
}
#[test]
fn disabled_account_should_not_be_schedulable() {
let account = account("acct_disabled").with_account_facts(
false,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
);
assert_eq!(
account.status_projection(SystemTime::now(), None).status,
AccountStatus::Disabled
);
}
#[test]
fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
let now = SystemTime::now();
let active_until = now + Duration::from_secs(60);
let account = account("acct_rate_limited");
let active = account.status_projection(now, Some(active_until));
assert_eq!(active.status, AccountStatus::RateLimited);
assert_eq!(active.rate_limited_until, Some(active_until));
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
assert_eq!(elapsed.status, AccountStatus::Normal);
assert_eq!(elapsed.rate_limited_until, None);
}
#[test]
fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
let account = ProviderAccount::new(
ProviderAccountId::new("acct_pending_identity").expect("valid account"),
ProviderKind::new("openai").expect("valid provider"),
"pending identity".to_owned(),
None,
"oauth".to_owned(),
CredentialRevision::new(1).expect("valid revision"),
Some(SystemTime::now() + Duration::from_secs(3600)),
)
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
);
assert_eq!(
(
account.credential_state(),
account.status_projection(SystemTime::now(), None).status
),
(CredentialState::Unknown, AccountStatus::Error)
);
}
#[test]
@@ -173,6 +357,84 @@ fn smart_selector_should_prefer_lower_inflight_count() {
assert_eq!(selected.candidate().account.id().as_str(), "acct_idle");
}
#[test]
fn smart_selector_should_prefer_lower_capacity_utilization_over_lower_inflight_count() {
let candidates = [
candidate_with_concurrency("acct_smaller", 1, 2),
candidate_with_concurrency("acct_larger", 2, 10),
];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_larger");
}
#[test]
fn selector_should_prioritize_the_highest_weight_for_every_rotation_strategy() {
let candidates = vec![
weighted_candidate("acct_low_weight", 1, 0),
weighted_candidate("acct_high_weight", 100, 2),
];
for strategy in [
RotationStrategy::Smart,
RotationStrategy::QuotaResetPriority,
RotationStrategy::RoundRobin,
RotationStrategy::Sticky,
] {
let selected = AccountSelector
.select(&candidates, &context(strategy))
.expect("highest weight account should be eligible");
assert_eq!(
selected.candidate().account.id().as_str(),
"acct_high_weight",
"strategy {strategy:?} must select inside the highest weight tier"
);
}
}
#[test]
fn selector_should_not_let_soft_affinity_bypass_a_higher_weight_account() {
let preferred = weighted_candidate("acct_preferred", 1, 0);
let preferred_id = preferred.account.id().clone();
let higher = weighted_candidate("acct_higher", 10, 0);
let mut selection = context(RotationStrategy::Smart);
selection.preferred_account = Some(preferred_id);
let candidates = [preferred, higher];
let selected = AccountSelector
.select(&candidates, &selection)
.expect("higher weight account should be eligible");
assert_eq!(selected.candidate().account.id().as_str(), "acct_higher");
assert_eq!(
selected.preferred(),
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::LowerWeight)
);
}
#[test]
fn selector_should_fall_back_when_a_higher_weight_account_reaches_its_override() {
let mut saturated = weighted_candidate("acct_high_weight", 100, 1);
saturated.account = saturated.account.with_scheduling(
Some(AccountConcurrencyLimit::new(1).expect("concurrency override")),
AccountWeight::new(100).expect("weight"),
);
let fallback = weighted_candidate("acct_low_weight", 1, 0);
let candidates = [saturated, fallback];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("lower weight account should be used after saturation");
assert_eq!(
selected.candidate().account.id().as_str(),
"acct_low_weight"
);
}
#[test]
fn smart_selector_should_balance_signals_instead_of_using_lexicographic_load() {
let mut healthy = candidate("acct_healthy", 1, Some(100));
@@ -286,15 +548,11 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
last_started_at: Some(last_started_at),
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
}
.with_provider_quota(Some(AccountQuotaSignals::new(
Some(reset_at),
Some(75),
false,
)));
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
assert_eq!(
(
@@ -387,6 +645,22 @@ fn quota_reset_selector_should_prefer_known_earliest_window() {
assert_eq!(selected.candidate().account.id().as_str(), "acct_earlier");
}
#[test]
fn quota_reset_selector_should_use_capacity_utilization_as_load_tiebreaker() {
let reset_at = SystemTime::now() + Duration::from_secs(60);
let mut smaller = candidate_with_concurrency("acct_smaller", 1, 2);
smaller.signals.quota_reset_at = Some(reset_at);
let mut larger = candidate_with_concurrency("acct_larger", 2, 10);
larger.signals.quota_reset_at = Some(reset_at);
let candidates = [smaller, larger];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::QuotaResetPriority))
.expect("candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_larger");
}
#[test]
fn smart_selector_should_prefer_known_quota_over_unknown_after_load_is_equal() {
let candidates = vec![
@@ -611,21 +885,31 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
}
#[test]
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
// availability 终态(Banned/Expired/Invalid)始终排除,
// 不受 quota 信号或 skip_exhausted 放宽影响。
fn enforced_eligibility_excludes_credential_errors_despite_quota_signals() {
let mut banned = candidate("acct_banned", 0, Some(100));
banned.account = banned
.account
.with_runtime_state(true, AccountAvailability::Banned);
banned.account = banned.account.with_account_facts(
true,
CredentialState::Banned,
QuotaState::unknown(),
None,
None,
);
let mut expired = candidate("acct_expired", 0, Some(100));
expired.account = expired
.account
.with_runtime_state(true, AccountAvailability::Expired);
expired.account = expired.account.with_account_facts(
true,
CredentialState::Expired,
QuotaState::unknown(),
None,
None,
);
let mut invalid = candidate("acct_invalid", 0, Some(100));
invalid.account = invalid
.account
.with_runtime_state(true, AccountAvailability::Invalid);
invalid.account = invalid.account.with_account_facts(
true,
CredentialState::Invalid,
QuotaState::unknown(),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![banned, expired, invalid, healthy];
@@ -637,14 +921,15 @@ fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
}
#[test]
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
fn enforced_eligibility_excludes_authoritative_quota_exhaustion() {
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
exhausted.account = exhausted
.account
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
exhausted.account = exhausted.account.with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![exhausted, healthy];
@@ -654,3 +939,31 @@ fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection()
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
#[test]
fn elapsed_quota_reset_does_not_fabricate_recovery() {
let now = SystemTime::now();
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
exhausted.account = exhausted.account.with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
now - Duration::from_secs(2),
Some(now - Duration::from_secs(1)),
),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
assert_eq!(
exhausted.account.status_projection(now, None).status,
AccountStatus::QuotaExhausted
);
let candidates = [exhausted, healthy];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("healthy candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
@@ -1,5 +1,5 @@
use std::{
collections::BTreeSet,
collections::{BTreeMap, BTreeSet},
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -18,17 +18,17 @@ use gateway_core::engine::continuation::{
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
};
use gateway_core::engine::credential::{
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
AccountSelectionPolicy, CredentialRevision, ProviderAccountId, RotationStrategy,
};
use gateway_core::engine::execution::{
ClientApiKeyUsageSink, ClientTransport, DefaultExecutionService, ExecutionRequestMetadata,
ExecutionService, ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
StartExecution, provider_failure_affects_circuit,
StartExecution, StartProviderExecution, provider_failure_affects_circuit,
};
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
use gateway_core::engine::provider::{
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
ProviderRequest, ProviderStream,
Provider, ProviderCallMetadata, ProviderCatalogGeneration, ProviderModelCapabilities,
ProviderRegistry, ProviderRequest, ProviderResource, ProviderStream, UpstreamTransport,
};
use gateway_core::engine::{
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
@@ -38,11 +38,15 @@ use gateway_core::error::{
ClientVisibleUpstreamResponse, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError,
StoreErrorKind,
};
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
use gateway_core::operation::{
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind, ProtocolPayload,
RawJsonPayload,
};
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
use gateway_core::routing::{
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RuntimeSnapshot,
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ProviderKind,
ProviderModel, PublicModelId, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot,
UpstreamModelId,
};
use serde_json::json;
@@ -192,6 +196,50 @@ impl Provider for FailingProvider {
}
}
struct ColdFailingProvider;
#[async_trait]
impl Provider for ColdFailingProvider {
fn name(&self) -> &'static str {
"openai"
}
fn catalog_generation(&self) -> ProviderCatalogGeneration {
ProviderCatalogGeneration::default()
}
async fn query_model_capabilities(
&self,
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
Ok(Vec::new())
}
async fn execute(
&self,
request: ProviderRequest,
_: AttemptContext,
) -> Result<ProviderStream, ProviderError> {
let metadata = ProviderCallMetadata::for_provider_endpoint(
request.candidate().provider().clone(),
ProviderResource::Account {
id: ProviderAccountId::new("acct_usage").expect("account"),
revision: CredentialRevision::new(1).expect("revision"),
},
UpstreamTransport::new("http_json").expect("transport"),
);
Ok(ProviderStream::new(
metadata,
futures::stream::once(async {
Err(ProviderError::new(
ProviderErrorKind::Transport,
UpstreamSendState::NotSent,
))
}),
(),
))
}
}
#[test]
fn successful_authentication_should_record_client_key_usage() {
let usage = Arc::new(RecordingClientApiKeyUsage::default());
@@ -212,6 +260,60 @@ fn successful_authentication_should_record_client_key_usage() {
assert_eq!(usage.recorded(), vec!["key_usage_test".to_owned()]);
}
#[test]
fn provider_endpoint_should_persist_its_real_v1_endpoint() {
let store = Arc::new(TrackingExecutionStore::default());
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(client_snapshot()),
store.clone(),
ProviderRegistry::new([Arc::new(ColdFailingProvider) as Arc<dyn Provider>])
.expect("provider registry"),
Arc::new(UnusedAdmissions),
Arc::new(UnusedCircuits),
Arc::new(UnusedContinuation),
Arc::new(RecordingClientApiKeyUsage::default()),
);
let client = service
.authenticate("sk_usage_test")
.expect("authenticated client");
let operation = Operation::GenerateImage(ImageRequest::from_raw_json(
ImageRequestKind::Generation,
RawJsonPayload::new(
"openai",
Bytes::from_static(br#"{"model":"gpt-image-2","prompt":"hello"}"#),
)
.expect("image payload"),
));
let mut started = block_on(service.start_provider_endpoint(StartProviderExecution {
client,
provider: ProviderKind::new("openai").expect("provider"),
operation,
metadata: ExecutionRequestMetadata {
protocol: "openai".to_owned(),
endpoint: "/v1/images/generations".to_owned(),
transport: ClientTransport::HttpJson,
stream: false,
client_ip: None,
user_agent: None,
previous_response_id: None,
},
}))
.expect("provider endpoint request should start without a text catalog entry");
let error = block_on(started.session.collect_uncommitted())
.expect_err("the cold provider stops execution after persistence");
assert_eq!(
store.model_request_endpoints(),
vec!["/v1/images/generations".to_owned()],
"execution error: {error:?}"
);
assert_eq!(store.requested_models(), vec![None]);
let upstream_models = store.upstream_models();
assert!(!upstream_models.is_empty());
assert!(upstream_models.iter().all(Option::is_none));
}
#[test]
fn circuit_store_failure_should_fail_open_during_request_start() {
let service = DefaultExecutionService::new(
@@ -283,7 +385,7 @@ fn slow_circuit_store_should_time_out_and_fail_open_during_request_start() {
}
#[test]
fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() {
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(start_snapshot()),
Arc::new(TrackingExecutionStore::default()),
@@ -298,7 +400,7 @@ fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
.expect("authenticated client");
let model = "gpt-future-not-in-catalog";
let started = block_on(service.start(StartExecution {
let result = block_on(service.start(StartExecution {
client,
public_model: PublicModelId::from_client_wire(model).expect("client model"),
operation: start_operation_for_model(model),
@@ -311,10 +413,80 @@ fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
user_agent: None,
previous_response_id: None,
},
}))
.expect("stale model catalog must not reject the request");
}));
let Err(error) = result else {
panic!("a known provider catalog must be authoritative for model availability");
};
assert!(!started.session.is_finalized());
assert_eq!(error.kind(), GatewayErrorKind::NoAvailableProvider);
}
#[test]
fn continuation_owned_by_another_client_api_key_should_fail_closed() {
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(start_snapshot()),
Arc::new(TrackingExecutionStore::default()),
ProviderRegistry::default(),
Arc::new(UnusedAdmissions),
Arc::new(UnusedCircuits),
Arc::new(RejectedContinuation::OwnershipMismatch),
Arc::new(RecordingClientApiKeyUsage::default()),
);
let client = service
.authenticate("sk_start_test")
.expect("authenticated client");
let result = block_on(service.start(StartExecution {
client,
public_model: PublicModelId::new("gpt-start").expect("public model"),
operation: start_operation(),
metadata: execution_metadata_with_continuation(),
}));
let Err(error) = result else {
panic!("cross-client continuation must be rejected");
};
assert_eq!(error.kind(), GatewayErrorKind::PolicyDenied);
}
#[test]
fn invalid_continuation_record_should_not_be_forwarded_as_an_external_handle() {
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(start_snapshot()),
Arc::new(TrackingExecutionStore::default()),
ProviderRegistry::default(),
Arc::new(UnusedAdmissions),
Arc::new(UnusedCircuits),
Arc::new(RejectedContinuation::InvalidData),
Arc::new(RecordingClientApiKeyUsage::default()),
);
let client = service
.authenticate("sk_start_test")
.expect("authenticated client");
let result = block_on(service.start(StartExecution {
client,
public_model: PublicModelId::new("gpt-start").expect("public model"),
operation: start_operation(),
metadata: execution_metadata_with_continuation(),
}));
let Err(error) = result else {
panic!("invalid continuation state must be rejected");
};
assert_eq!(error.kind(), GatewayErrorKind::Internal);
}
fn execution_metadata_with_continuation() -> ExecutionRequestMetadata {
ExecutionRequestMetadata {
protocol: "openai".to_owned(),
endpoint: "/v1/responses".to_owned(),
transport: ClientTransport::HttpJson,
stream: false,
client_ip: None,
user_agent: None,
previous_response_id: Some(PreviousResponseId::new("response-private")),
}
}
#[derive(Default)]
@@ -341,6 +513,9 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
struct TrackingExecutionStore {
touched: AtomicBool,
probe_failures: Mutex<Vec<String>>,
model_request_endpoints: Mutex<Vec<String>>,
requested_models: Mutex<Vec<Option<String>>>,
upstream_models: Mutex<Vec<Option<String>>>,
fail_probe_observation: AtomicBool,
}
@@ -355,17 +530,60 @@ impl TrackingExecutionStore {
.expect("probe failures lock")
.clone()
}
fn model_request_endpoints(&self) -> Vec<String> {
self.model_request_endpoints
.lock()
.expect("model request endpoints lock")
.clone()
}
fn requested_models(&self) -> Vec<Option<String>> {
self.requested_models
.lock()
.expect("requested models lock")
.clone()
}
fn upstream_models(&self) -> Vec<Option<String>> {
self.upstream_models
.lock()
.expect("upstream models lock")
.clone()
}
}
#[async_trait]
impl ExecutionStore for TrackingExecutionStore {
async fn create_model_request(&self, _: NewModelRequest) -> Result<(), StoreError> {
async fn create_model_request(&self, request: NewModelRequest) -> Result<(), StoreError> {
self.touch();
self.requested_models
.lock()
.expect("requested models lock")
.push(
request
.requested_model
.as_ref()
.map(|model| model.as_str().to_owned()),
);
self.model_request_endpoints
.lock()
.expect("model request endpoints lock")
.push(request.endpoint);
Ok(())
}
async fn record_attempt(&self, _: AttemptRecord) -> Result<(), StoreError> {
async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError> {
self.touch();
self.upstream_models
.lock()
.expect("upstream models lock")
.push(
attempt
.upstream_model_id
.as_ref()
.map(|model| model.as_str().to_owned()),
);
Ok(())
}
@@ -526,6 +744,7 @@ struct UnusedContinuation;
impl NativeContinuationPort for UnusedContinuation {
fn resolve<'a>(
&'a self,
_: &'a ClientApiKeyId,
_: &'a PreviousResponseId,
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
Box::pin(async { Ok(None) })
@@ -539,6 +758,43 @@ impl NativeContinuationPort for UnusedContinuation {
}
}
enum RejectedContinuation {
OwnershipMismatch,
InvalidData,
}
impl NativeContinuationPort for RejectedContinuation {
fn resolve<'a>(
&'a self,
_: &'a ClientApiKeyId,
_: &'a PreviousResponseId,
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
Box::pin(async move {
Err(match self {
Self::OwnershipMismatch => NativeContinuationStoreError::ownership_mismatch(),
Self::InvalidData => NativeContinuationStoreError::invalid_data("invalid record"),
})
})
}
fn record<'a>(
&'a self,
_: NativeContinuationPin,
) -> BoxFuture<'a, Result<(), NativeContinuationStoreError>> {
Box::pin(async { Ok(()) })
}
}
fn account_scope(provider: &ProviderKind, account_id: &str) -> Arc<FrozenAccountScope> {
Arc::new(FrozenAccountScope::new(
Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
ProviderAccountId::new(account_id).expect("account ID"),
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
)]))),
ClientRoutingScope::all_accounts(),
))
}
fn probe_snapshot() -> RuntimeSnapshot {
let provider = ProviderKind::new("openai").expect("provider kind");
let capabilities =
@@ -575,7 +831,7 @@ fn client_snapshot() -> RuntimeSnapshot {
vec![ClientPolicy::new(
ClientApiKeyId::new("key_usage_test").expect("client API key ID"),
PlaintextClientApiKey::new("sk_usage_test").expect("plaintext client API key"),
provider,
account_scope(&provider, "acct_usage"),
true,
RateLimits::unlimited(),
)],
@@ -603,7 +859,7 @@ fn start_snapshot() -> RuntimeSnapshot {
vec![ClientPolicy::new(
ClientApiKeyId::new("key_start_test").expect("client API key ID"),
PlaintextClientApiKey::new("sk_start_test").expect("plaintext client API key"),
provider,
account_scope(&provider, "acct_start"),
true,
RateLimits::unlimited(),
)],
@@ -77,6 +77,17 @@ fn protocol_wire_event_should_preserve_opaque_sse_metadata() {
assert_eq!(wire.sse_id(), Some(sse_id.as_str()));
}
#[test]
fn protocol_wire_event_should_preserve_raw_json_body_without_parsing_it() {
let raw = Bytes::from_static(
br#"{ "created": 1, "data": [{"b64_json":"AAAA"}], "future": 9007199254740993 }"#,
);
let wire = ProtocolWireEvent::raw_json("openai", raw.clone()).expect("raw JSON event");
assert_eq!(wire.raw_json_body(), Some(&raw));
assert_eq!(wire.into_raw_json_body(), Some(raw));
}
#[test]
fn validator_should_accept_started_content_delta_completed_sequence() {
let mut validator = EventSequenceValidator::new();
@@ -1,7 +1,9 @@
use bytes::Bytes;
use gateway_core::error::OperationError;
use gateway_core::operation::{
EmbedRequest, Feature, GenerateRequest, ImageRequest, Operation, OperationKind,
ProtocolPayload, ProviderSessionState, RerankRequest, RetrySafety, SpeechRequest,
EmbedRequest, Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, RerankRequest,
RetrySafety, SpeechRequest,
};
use serde_json::{Map, Value, json};
@@ -12,6 +14,17 @@ fn generate(body: Value) -> GenerateRequest {
)
}
fn image(kind: ImageRequestKind, body: Value) -> ImageRequest {
ImageRequest::from_raw_json(
kind,
RawJsonPayload::new(
"openai",
Bytes::from(serde_json::to_vec(&body).expect("image JSON")),
)
.expect("OpenAI payload"),
)
}
#[test]
fn generate_request_should_keep_client_body_opaque_and_redacted() {
let secret = "private prompt body";
@@ -113,10 +126,6 @@ fn non_generate_requests_should_reject_missing_required_fields() {
RerankRequest::new("", vec!["document".to_owned()]).expect_err("query is required"),
OperationError::EmptyField { field: "query" }
);
assert_eq!(
ImageRequest::new("").expect_err("prompt is required"),
OperationError::EmptyField { field: "prompt" }
);
assert_eq!(
SpeechRequest::new("hello", "").expect_err("voice is required"),
OperationError::EmptyField { field: "voice" }
@@ -132,7 +141,10 @@ fn operation_kind_and_retry_safety_should_remain_stable() {
let rerank = Operation::Rerank(
RerankRequest::new("query", vec!["document".to_owned()]).expect("rerank request is valid"),
);
let image = Operation::GenerateImage(ImageRequest::new("draw").expect("image request"));
let image = Operation::GenerateImage(image(
ImageRequestKind::Generation,
json!({"model": "gpt-image-2", "prompt": "draw"}),
));
let speech = Operation::Speech(SpeechRequest::new("hello", "alloy").expect("speech request"));
assert_eq!(generate.kind(), OperationKind::Generate);
@@ -140,5 +152,35 @@ fn operation_kind_and_retry_safety_should_remain_stable() {
assert_eq!(embed.retry_safety(), RetrySafety::Idempotent);
assert_eq!(rerank.retry_safety(), RetrySafety::Idempotent);
assert_eq!(image.kind(), OperationKind::GenerateImage);
assert!(image.image_generation_requested());
assert_eq!(speech.kind(), OperationKind::Speech);
}
#[test]
fn image_request_should_preserve_generation_and_edit_payloads_opaque() {
let generation_body = json!({
"model": "gpt-image-2",
"prompt": "draw",
"future_official_field": {"keep": true},
});
let edit_body = json!({
"model": "gpt-image-2",
"images": [{"image_url": "data:image/png;base64,AAAA"}],
"prompt": "edit",
"future_official_field": [1, 2, 3],
});
let generation = image(ImageRequestKind::Generation, generation_body.clone());
let edit = image(ImageRequestKind::Edit, edit_body.clone());
assert_eq!(generation.kind(), ImageRequestKind::Generation);
assert_eq!(edit.kind(), ImageRequestKind::Edit);
assert_eq!(
serde_json::from_slice::<Value>(generation.payload().body()).expect("generation JSON"),
generation_body
);
assert_eq!(
serde_json::from_slice::<Value>(edit.payload().body()).expect("edit JSON"),
edit_body
);
assert!(!format!("{edit:?}").contains("data:image/png"));
}
@@ -1,16 +1,25 @@
use std::sync::Arc;
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::ProviderKind;
use gateway_core::routing::{ClientRoutingScope, FrozenAccountScope, RuntimeAccountDirectory};
fn plaintext(value: &str) -> PlaintextClientApiKey {
PlaintextClientApiKey::new(value).expect("valid plaintext client key")
}
fn account_scope() -> Arc<FrozenAccountScope> {
Arc::new(FrozenAccountScope::new(
Arc::new(RuntimeAccountDirectory::default()),
ClientRoutingScope::all_accounts(),
))
}
#[test]
fn disabled_client_key_should_be_denied() {
let policy = ClientPolicy::new(
ClientApiKeyId::new("key_disabled").expect("valid key ID"),
plaintext("sk_disabled_secret"),
ProviderKind::new("openai").expect("valid provider"),
account_scope(),
false,
RateLimits::unlimited(),
);
@@ -23,7 +32,7 @@ fn enabled_client_key_should_be_authorized() {
let policy = ClientPolicy::new(
ClientApiKeyId::new("key_enabled").expect("valid key ID"),
plaintext("sk_enabled_secret"),
ProviderKind::new("openai").expect("valid provider"),
account_scope(),
true,
RateLimits::unlimited(),
);
@@ -57,8 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
}
#[test]
fn refresh_schedule_should_use_the_exact_configured_margin() {
let account = ProviderAccountId::new("acct_stable").expect("valid account");
fn refresh_policy_should_mark_tokens_due_at_the_exact_configured_margin() {
let policy = ProviderRefreshPolicy::try_new(
Duration::from_secs(3_600),
NonZeroU32::new(2).expect("positive concurrency"),
@@ -67,15 +66,12 @@ fn refresh_schedule_should_use_the_exact_configured_margin() {
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
let expires_at = observed_at + Duration::from_secs(7_200);
let first = policy
.next_attempt_at(&account, expires_at, observed_at)
.expect("schedule refresh");
assert_eq!(first, expires_at - Duration::from_secs(3_600));
assert!(!policy.is_refresh_due(expires_at, observed_at));
assert!(policy.is_refresh_due(observed_at + Duration::from_secs(3_600), observed_at));
}
#[test]
fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
let account = ProviderAccountId::new("acct_boundary").expect("valid account");
fn refresh_policy_should_mark_expired_tokens_due() {
let policy = ProviderRefreshPolicy::try_new(
Duration::from_secs(3_600),
NonZeroU32::new(1).expect("positive concurrency"),
@@ -83,12 +79,7 @@ fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
.expect("valid policy");
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
assert_eq!(
policy
.next_attempt_at(&account, observed_at + Duration::from_secs(1), observed_at,)
.expect("schedule immediate refresh"),
observed_at
);
assert!(policy.is_refresh_due(observed_at - Duration::from_secs(1), observed_at));
}
#[test]
@@ -101,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: Some(7),
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: Some(125),
first_output_latency_ms: Some(250),
},
+245 -19
View File
@@ -1,15 +1,22 @@
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::Duration;
use gateway_core::engine::credential::{AccountSelectionPolicy, RotationStrategy};
use bytes::Bytes;
use gateway_core::engine::credential::{
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
};
use gateway_core::operation::{
CapabilityRequirements, Feature, GenerateRequest, Operation, OperationKind, ProtocolPayload,
CapabilityRequirements, Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
OperationKind, ProtocolPayload, RawJsonPayload,
};
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::{
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
RuntimeSnapshot, UpstreamModelId,
AccountGroupId, AccountRoutingScopeKind, ClientRoutingScope, ConfigRevision,
FrozenAccountScope, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId,
RoutingContext, RoutingGroupSnapshot, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot,
UpstreamModelId,
};
mod snapshot;
@@ -45,11 +52,49 @@ fn operation() -> Operation {
))
}
fn image_operation() -> Operation {
let body = serde_json::json!({
"model": "gpt-image-2",
"prompt": "draw",
});
Operation::GenerateImage(ImageRequest::from_raw_json(
ImageRequestKind::Generation,
RawJsonPayload::new(
"openai",
Bytes::from(serde_json::to_vec(&body).expect("image JSON")),
)
.expect("OpenAI payload"),
))
}
fn account_directory() -> Arc<RuntimeAccountDirectory> {
Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([
(
ProviderAccountId::new("acct_openai").expect("account"),
RuntimeAccount::new(
ProviderKind::new("openai").expect("provider"),
BTreeSet::new(),
),
),
(
ProviderAccountId::new("acct_xai").expect("account"),
RuntimeAccount::new(ProviderKind::new("xai").expect("provider"), BTreeSet::new()),
),
])))
}
fn account_scope() -> Arc<FrozenAccountScope> {
Arc::new(FrozenAccountScope::new(
account_directory(),
ClientRoutingScope::all_accounts(),
))
}
fn client_policy(id: &str, plaintext: &str, enabled: bool) -> ClientPolicy {
ClientPolicy::new(
ClientApiKeyId::new(id).expect("client key ID"),
PlaintextClientApiKey::new(plaintext).expect("plaintext client key"),
ProviderKind::new("openai").expect("valid provider"),
account_scope(),
enabled,
RateLimits::unlimited(),
)
@@ -70,6 +115,7 @@ fn snapshot() -> RuntimeSnapshot {
Vec::new(),
)
.expect("snapshot")
.with_account_directory(account_directory())
.with_model_mappings(BTreeMap::from([
("gpt-5.4".to_owned(), "gpt-5.5".to_owned()),
("grok-latest".to_owned(), "grok-4.5".to_owned()),
@@ -81,6 +127,92 @@ fn config_revision_should_reject_zero() {
assert!(ConfigRevision::new(0).is_err());
}
#[test]
fn account_group_id_should_match_the_database_contract_exactly() {
let valid = "grp_0123456789abcdef0123456789abcdef";
assert_eq!(
AccountGroupId::new(valid).expect("valid group ID").as_str(),
valid
);
for invalid in [
"0123456789abcdef0123456789abcdef",
"grp_0123456789abcdef0123456789abcde",
"grp_0123456789abcdef0123456789abcdef0",
"grp_0123456789abcdef0123456789abcdeF",
"grp_0123456789abcdef0123456789abcdeg",
] {
assert!(
AccountGroupId::new(invalid).is_err(),
"invalid group ID was accepted: {invalid}"
);
}
}
#[test]
fn restricted_scope_should_use_enabled_group_union_and_preserve_all_bindings_for_history() {
let enabled =
AccountGroupId::new("grp_11111111111111111111111111111111").expect("enabled group");
let disabled =
AccountGroupId::new("grp_22222222222222222222222222222222").expect("disabled group");
let provider = ProviderKind::new("openai").expect("provider");
let allowed_account = ProviderAccountId::new("acct_allowed").expect("account");
let disabled_only_account = ProviderAccountId::new("acct_disabled").expect("account");
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([
(
allowed_account.clone(),
RuntimeAccount::new(provider.clone(), BTreeSet::from([enabled.clone()])),
),
(
disabled_only_account.clone(),
RuntimeAccount::new(provider.clone(), BTreeSet::from([disabled.clone()])),
),
])));
let scope = FrozenAccountScope::new(
directory,
ClientRoutingScope::restricted(
vec![
RoutingGroupSnapshot::new(enabled.clone(), "Enabled".to_owned()),
RoutingGroupSnapshot::new(disabled, "Disabled".to_owned()),
],
BTreeSet::from([enabled]),
BTreeSet::from([provider]),
)
.expect("restricted scope"),
);
assert!(scope.allows(&allowed_account));
assert!(!scope.allows(&disabled_only_account));
let history = scope.routing_snapshot();
assert_eq!(history.kind(), AccountRoutingScopeKind::Groups);
assert_eq!(history.groups_snapshot().len(), 2);
assert_eq!(history.groups_snapshot()[1].name(), "Disabled");
}
#[test]
fn restricted_scope_with_no_enabled_group_should_fail_closed() {
let group = AccountGroupId::new("grp_33333333333333333333333333333333").expect("group");
let account = ProviderAccountId::new("acct_group_member").expect("account");
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
account.clone(),
RuntimeAccount::new(
ProviderKind::new("openai").expect("provider"),
BTreeSet::from([group.clone()]),
),
)])));
let scope = FrozenAccountScope::new(
directory,
ClientRoutingScope::restricted(
vec![RoutingGroupSnapshot::new(group, "Disabled".to_owned())],
BTreeSet::new(),
BTreeSet::new(),
)
.expect("restricted scope"),
);
assert!(scope.provider_kinds().is_empty());
assert!(!scope.allows(&account));
}
#[test]
fn snapshot_should_publish_only_enabled_plaintext_client_policies() {
let snapshot = RuntimeSnapshot::new(
@@ -98,7 +230,12 @@ fn snapshot_should_publish_only_enabled_plaintext_client_policies() {
let policies = snapshot.client_policies().collect::<Vec<_>>();
assert_eq!(policies.len(), 1);
assert_eq!(policies[0].key_id().as_str(), "key_enabled");
assert_eq!(policies[0].provider_kind().as_str(), "openai");
assert!(
policies[0]
.account_scope()
.provider_kinds()
.contains(&ProviderKind::new("openai").expect("provider"))
);
}
#[test]
@@ -137,8 +274,9 @@ fn selected_provider_should_use_global_model_mapping() {
.plan(
&PublicModelId::new("gpt-5.4").expect("model"),
&operation(),
snapshot.all_account_scope(),
&RoutingContext {
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
required_provider: Some(ProviderKind::new("openai").expect("provider")),
..RoutingContext::default()
},
)
@@ -146,7 +284,13 @@ fn selected_provider_should_use_global_model_mapping() {
assert_eq!(plan.candidates().len(), 1);
assert_eq!(plan.candidates()[0].provider().as_str(), "openai");
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "gpt-5.5");
assert_eq!(
plan.candidates()[0]
.upstream_model()
.expect("model route candidate")
.as_str(),
"gpt-5.5"
);
}
#[test]
@@ -170,26 +314,82 @@ fn cyclic_model_mapping_should_fall_back_to_the_original_name() {
}
#[test]
fn global_mapping_should_apply_to_every_provider() {
fn known_provider_catalog_missing_mapped_model_should_be_filtered() {
let snapshot = snapshot();
let plan = snapshot
let error = snapshot
.plan(
&PublicModelId::new("gpt-5.4").expect("model"),
&operation(),
snapshot.all_account_scope(),
&RoutingContext {
provider_kind: Some(ProviderKind::new("xai").expect("provider")),
required_provider: Some(ProviderKind::new("xai").expect("provider")),
..RoutingContext::default()
},
)
.expect("transparent plan");
.expect_err("known xAI catalog does not contain the mapped OpenAI model");
assert_eq!(plan.candidates()[0].provider().as_str(), "xai");
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "gpt-5.5");
assert!(matches!(
error,
gateway_core::error::RoutingError::NoCapableProvider { .. }
));
}
#[test]
fn provider_endpoint_plan_should_not_consult_or_publish_the_text_model_catalog() {
let snapshot = snapshot();
let provider = ProviderKind::new("openai").expect("provider");
let public_model = PublicModelId::new("gpt-image-2").expect("model");
assert!(!snapshot.contains_public_model_for_provider(&public_model, &provider));
let plan = snapshot
.plan_provider_endpoint(
&provider,
&image_operation(),
snapshot.all_account_scope(),
&RoutingContext::default(),
)
.expect("provider-owned endpoint remains routable");
assert_eq!(plan.operation(), OperationKind::GenerateImage);
assert_eq!(plan.candidates().len(), 1);
assert_eq!(plan.candidates()[0].provider(), &provider);
assert_eq!(plan.candidates()[0].upstream_model(), None);
assert!(!snapshot.contains_public_model_for_provider(&public_model, &provider));
}
#[test]
fn provider_endpoint_plan_should_still_respect_circuit_filtering() {
let snapshot = snapshot();
let provider = ProviderKind::new("openai").expect("provider");
let error = snapshot
.plan_provider_endpoint(
&provider,
&image_operation(),
snapshot.all_account_scope(),
&RoutingContext {
blocked_providers: BTreeSet::from([provider.clone()]),
..RoutingContext::default()
},
)
.expect_err("blocked provider has no endpoint candidate");
assert!(matches!(
error,
gateway_core::error::RoutingError::NoCapableProviderEndpoint { .. }
));
}
#[test]
fn unmapped_model_should_pass_through_unchanged() {
let snapshot = snapshot();
let snapshot = RuntimeSnapshot::new(
ConfigRevision::new(1).expect("revision"),
scheduling(),
vec![ProviderKind::new("openai").expect("provider")],
Vec::new(),
Vec::new(),
)
.expect("snapshot")
.with_account_directory(account_directory());
for requested in [
"future-openai-model".to_owned(),
format!("future-{}", "x".repeat(512)),
@@ -200,14 +400,21 @@ fn unmapped_model_should_pass_through_unchanged() {
.plan(
&PublicModelId::from_client_wire(requested.clone()).expect("client model"),
&operation(),
snapshot.all_account_scope(),
&RoutingContext {
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
required_provider: Some(ProviderKind::new("openai").expect("provider")),
..RoutingContext::default()
},
)
.expect("unknown model remains transparent");
assert_eq!(plan.candidates()[0].upstream_model().as_str(), requested);
assert_eq!(
plan.candidates()[0]
.upstream_model()
.expect("model route candidate")
.as_str(),
requested
);
}
}
@@ -229,8 +436,9 @@ fn blocked_provider_should_be_filtered() {
.plan(
&PublicModelId::new("gpt-5.5").expect("model"),
&operation(),
snapshot.all_account_scope(),
&RoutingContext {
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
required_provider: Some(ProviderKind::new("openai").expect("provider")),
blocked_providers: BTreeSet::from([ProviderKind::new("openai").expect("provider")]),
},
)
@@ -262,8 +470,9 @@ fn known_unsupported_operation_should_not_be_bypassed() {
.plan(
&PublicModelId::new("gpt-known-unsupported").expect("model"),
&operation(),
snapshot.all_account_scope(),
&RoutingContext {
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
required_provider: Some(ProviderKind::new("openai").expect("provider")),
..RoutingContext::default()
},
)
@@ -316,3 +525,20 @@ fn unknown_model_should_be_hidden_when_provider_catalog_is_available() {
&ProviderKind::new("openai").expect("provider"),
));
}
#[test]
fn alias_should_only_be_available_from_a_provider_with_its_mapped_model() {
let snapshot = snapshot();
let alias = PublicModelId::new("grok-latest").expect("model");
assert!(!snapshot.contains_public_model_for_provider(
&alias,
&ProviderKind::new("openai").expect("provider"),
));
assert!(
snapshot.contains_public_model_for_provider(
&alias,
&ProviderKind::new("xai").expect("provider"),
)
);
}
@@ -19,7 +19,8 @@ use gateway_core::operation::OperationKind;
use gateway_core::policy::{ClientApiKeyId, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::snapshot::{
RuntimeSnapshotCompileError, RuntimeSnapshotCompiler, RuntimeSnapshotHandle,
RuntimeSnapshotPublisher, SnapshotClientPolicyFacts, SnapshotControl, SnapshotFacts,
RuntimeSnapshotPublisher, SnapshotAccountGroupFacts, SnapshotAccountGroupMemberFacts,
SnapshotClientPolicyFacts, SnapshotControl, SnapshotFacts, SnapshotProviderAccountFacts,
SnapshotRevisionStream, SnapshotSettingsFacts, SnapshotStoreError, SnapshotStorePort,
SnapshotSubscriptionError, SnapshotSubscriptionPort, runtime_revision_needs_refresh,
};
@@ -348,9 +349,12 @@ fn facts(config_revision: u64, observed_current_revision: u64) -> SnapshotFacts
vec![SnapshotClientPolicyFacts::new(
ClientApiKeyId::new("key_one").expect("key ID"),
PlaintextClientApiKey::new("sk_test").expect("plaintext key"),
"alpha",
Vec::new(),
RateLimits::unlimited(),
)],
Vec::<SnapshotAccountGroupFacts>::new(),
Vec::<SnapshotProviderAccountFacts>::new(),
Vec::<SnapshotAccountGroupMemberFacts>::new(),
)
}
+12 -1
View File
@@ -41,6 +41,7 @@ pub fn load_config<T: LoadableConfig>() -> Result<T, ConfigError> {
#[serde(deny_unknown_fields)]
pub struct HostConfig {
pub listen: ListenConfig,
pub runtime_data_dir: PathBuf,
pub logging: LoggingConfig,
#[serde(default)]
pub system_update: SystemUpdateConfig,
@@ -76,11 +77,21 @@ impl HostConfig {
"host.worker_shutdown_timeout_seconds",
));
}
if self.runtime_data_dir.as_os_str().is_empty() {
return Err(ConfigError::InvalidField("host.runtime_data_dir"));
}
resolve_relative_path(source_dir, &mut self.runtime_data_dir);
self.logging.resolve_and_validate(source_dir)?;
self.system_update.resolve_and_validate(source_dir)?;
self.system_update
.resolve_and_validate(source_dir, &self.runtime_data_dir)?;
Ok(())
}
#[must_use]
pub fn runtime_data_dir(&self) -> &Path {
&self.runtime_data_dir
}
#[must_use]
pub const fn drain_timeout(&self) -> Duration {
Duration::from_secs(self.drain_timeout_seconds)
+80 -23
View File
@@ -13,6 +13,9 @@ use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::Subscriber
use crate::config::LoggingConfig;
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
/// non-blocking 日志 writer 的进程级守卫。
pub struct LogGuard {
@@ -34,7 +37,8 @@ pub enum LogError {
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
let file_filter = EnvFilter::try_new(&directive).map_err(|_| LogError::InvalidFilter)?;
let file_filter = application_file_filter(&directive)?;
let recovery_file_filter = oauth_recovery_file_filter();
let stdout_filter =
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
.map_err(|_| LogError::InvalidFilter)?;
@@ -47,25 +51,37 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
guards.push(guard);
writer
});
let file_writer = if config.file.enabled {
let (file_writer, recovery_file_writer) = if config.file.enabled {
let maximum_bytes = config
.file
.max_file_size_mb
.checked_mul(1024 * 1024)
.ok_or(LogError::SizeOverflow)?;
let writer = RotatingLogWriter::open(
let file_writer = RotatingLogWriter::open(
config.file.directory.clone(),
LOG_FILE_PREFIX,
maximum_bytes,
config.file.retention_days,
config.file.max_files,
)?;
let (writer, guard) = NonBlockingBuilder::default()
let recovery_file_writer = RotatingLogWriter::open(
config.file.directory.clone(),
OAUTH_RECOVERY_LOG_FILE_PREFIX,
maximum_bytes,
config.file.retention_days,
config.file.max_files,
)?;
let (file_writer, file_guard) = NonBlockingBuilder::default()
.thread_name("gateway-log-file")
.finish(writer);
guards.push(guard);
Some(writer)
.finish(file_writer);
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
.thread_name("gateway-log-oauth-recovery-file")
.finish(recovery_file_writer);
guards.push(file_guard);
guards.push(recovery_file_guard);
(Some(file_writer), Some(recovery_file_writer))
} else {
None
(None, None)
};
let stdout_layer = stdout_writer.map(|writer| {
@@ -89,14 +105,43 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
.with_span_list(true)
.with_filter(file_filter)
});
let recovery_file_layer = recovery_file_writer.map(|writer| {
tracing_subscriber::fmt::layer()
.json()
.with_writer(writer)
.with_target(true)
.with_file(true)
.with_line_number(true)
.with_thread_ids(true)
.with_thread_names(true)
.with_current_span(true)
.with_span_list(true)
.with_filter(recovery_file_filter)
});
tracing_subscriber::registry()
.with(stdout_layer)
.with(file_layer)
.with(recovery_file_layer)
.try_init()
.map_err(|_| LogError::AlreadyInitialized)?;
Ok(LogGuard { _writers: guards })
}
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
.parse()
.expect("static OAuth recovery log directive is valid");
EnvFilter::try_new(directive)
.map(|filter| filter.add_directive(recovery_directive))
.map_err(|_| LogError::InvalidFilter)
}
fn oauth_recovery_file_filter() -> EnvFilter {
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
.parse()
.expect("static OAuth recovery log filter is valid")
}
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
if !persistent_log_enabled {
directive.to_owned()
@@ -107,6 +152,7 @@ fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> Str
struct RotatingLogWriter {
directory: PathBuf,
file_prefix: &'static str,
maximum_bytes: u64,
retention_days: usize,
maximum_files: usize,
@@ -119,17 +165,20 @@ struct RotatingLogWriter {
impl RotatingLogWriter {
fn open(
directory: PathBuf,
file_prefix: &'static str,
maximum_bytes: u64,
retention_days: usize,
maximum_files: usize,
) -> io::Result<Self> {
fs::create_dir_all(&directory)?;
let date = Utc::now().date_naive();
cleanup_log_files(&directory, date, retention_days, maximum_files)?;
let (segment, bytes_written) = writable_log_segment(&directory, date, maximum_bytes)?;
let file = open_log_segment(&directory, date, segment)?;
cleanup_log_files(&directory, file_prefix, date, retention_days, maximum_files)?;
let (segment, bytes_written) =
writable_log_segment(&directory, file_prefix, date, maximum_bytes)?;
let file = open_log_segment(&directory, file_prefix, date, segment)?;
Ok(Self {
directory,
file_prefix,
maximum_bytes,
retention_days,
maximum_files,
@@ -157,10 +206,11 @@ impl RotatingLogWriter {
} else {
self.segment = self.segment.saturating_add(1);
}
self.file = open_log_segment(&self.directory, self.date, self.segment)?;
self.file = open_log_segment(&self.directory, self.file_prefix, self.date, self.segment)?;
self.bytes_written = self.file.metadata()?.len();
cleanup_log_files(
&self.directory,
self.file_prefix,
self.date,
self.retention_days,
self.maximum_files,
@@ -192,10 +242,11 @@ struct ManagedLogFile {
fn writable_log_segment(
directory: &Path,
file_prefix: &str,
date: chrono::NaiveDate,
maximum_bytes: u64,
) -> io::Result<(usize, u64)> {
let latest = managed_log_files(directory)?
let latest = managed_log_files(directory, file_prefix)?
.into_iter()
.filter(|entry| entry.date == date)
.max_by_key(|entry| entry.segment);
@@ -210,26 +261,31 @@ fn writable_log_segment(
}
}
fn open_log_segment(directory: &Path, date: chrono::NaiveDate, segment: usize) -> io::Result<File> {
fn open_log_segment(
directory: &Path,
file_prefix: &str,
date: chrono::NaiveDate,
segment: usize,
) -> io::Result<File> {
OpenOptions::new()
.create(true)
.append(true)
.open(directory.join(log_file_name(date, segment)))
.open(directory.join(log_file_name(file_prefix, date, segment)))
}
fn log_file_name(date: chrono::NaiveDate, segment: usize) -> String {
fn log_file_name(file_prefix: &str, date: chrono::NaiveDate, segment: usize) -> String {
if segment == 0 {
format!("{LOG_FILE_PREFIX}.{date}.log")
format!("{file_prefix}.{date}.log")
} else {
format!("{LOG_FILE_PREFIX}.{date}.{segment}.log")
format!("{file_prefix}.{date}.{segment}.log")
}
}
fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
fn managed_log_files(directory: &Path, file_prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
Ok(fs::read_dir(directory)?
.filter_map(Result::ok)
.filter_map(|entry| {
parse_log_file_name(entry.file_name().to_string_lossy().as_ref()).map(
parse_log_file_name(entry.file_name().to_string_lossy().as_ref(), file_prefix).map(
|(date, segment)| ManagedLogFile {
date,
segment,
@@ -240,9 +296,9 @@ fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
.collect())
}
fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
fn parse_log_file_name(name: &str, file_prefix: &str) -> Option<(chrono::NaiveDate, usize)> {
let body = name
.strip_prefix(&format!("{LOG_FILE_PREFIX}."))?
.strip_prefix(&format!("{file_prefix}."))?
.strip_suffix(".log")?;
let (date, segment) = body.split_once('.').map_or((body, 0), |(date, segment)| {
(date, segment.parse().ok().unwrap_or(usize::MAX))
@@ -257,6 +313,7 @@ fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
fn cleanup_log_files(
directory: &Path,
file_prefix: &str,
today: chrono::NaiveDate,
retention_days: usize,
maximum_files: usize,
@@ -266,7 +323,7 @@ fn cleanup_log_files(
.checked_sub_signed(chrono::Duration::days(retention_offset))
.unwrap_or(chrono::NaiveDate::MIN);
let mut retained = Vec::new();
for entry in managed_log_files(directory)? {
for entry in managed_log_files(directory, file_prefix)? {
if entry.date < cutoff {
fs::remove_file(entry.path)?;
} else {
@@ -81,10 +81,10 @@ impl Default for SystemUpdateConfig {
});
let update_state_file = environment_value("CPR_UPDATE_STATE_FILE")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("/app/.runtime/data/update-state.json"));
.unwrap_or_else(|| PathBuf::from("update-state.json"));
let update_lock_file = environment_value("CPR_UPDATE_LOCK_FILE")
.map(PathBuf::from)
.unwrap_or_else(|| update_state_file.with_extension("lock"));
.unwrap_or_else(|| update_state_file.with_file_name("update.lock"));
let update_temp_dir = environment_value("CPR_UPDATE_TEMP_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| state::default_temp_dir(&update_state_file));
@@ -120,21 +120,28 @@ impl Default for SystemUpdateConfig {
}
impl SystemUpdateConfig {
pub(crate) fn resolve_and_validate(&mut self, source_dir: &Path) -> Result<(), ConfigError> {
for path in [
self.executable_path.as_mut(),
Some(&mut self.web_dist_dir),
Some(&mut self.update_state_file),
Some(&mut self.update_lock_file),
Some(&mut self.update_temp_dir),
]
.into_iter()
.flatten()
pub(crate) fn resolve_and_validate(
&mut self,
source_dir: &Path,
runtime_data_dir: &Path,
) -> Result<(), ConfigError> {
for path in [self.executable_path.as_mut(), Some(&mut self.web_dist_dir)]
.into_iter()
.flatten()
{
if path.is_relative() {
*path = source_dir.join(&*path);
}
}
for path in [
&mut self.update_state_file,
&mut self.update_lock_file,
&mut self.update_temp_dir,
] {
if path.is_relative() {
*path = runtime_data_dir.join(&*path);
}
}
if self.version.trim().is_empty() {
return Err(ConfigError::InvalidField("host.system_update.version"));
}
+38 -3
View File
@@ -34,6 +34,40 @@ fn host_config_resolves_only_host_owned_relative_paths() {
);
}
#[test]
fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
let mut config = valid_config();
config
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
.expect("valid host config");
assert_eq!(
config.runtime_data_dir(),
PathBuf::from("/srv/gateway/runtime-data")
);
}
#[test]
fn host_config_derives_update_paths_from_runtime_data_dir() {
let mut config = valid_config();
config
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
.expect("valid host config");
assert_eq!(
(
config.system_update.update_state_file,
config.system_update.update_lock_file,
config.system_update.update_temp_dir,
),
(
PathBuf::from("/srv/gateway/runtime-data/update-state.json"),
PathBuf::from("/srv/gateway/runtime-data/update.lock"),
PathBuf::from("/srv/gateway/runtime-data/update-tmp"),
)
);
}
#[test]
fn host_config_rejects_zero_drain_window() {
let mut config = valid_config();
@@ -48,9 +82,9 @@ fn host_config_rejects_zero_drain_window() {
fn valid_config() -> HostConfig {
let system_update = SystemUpdateConfig {
update_state_file: PathBuf::from(".runtime/update-state.json"),
update_lock_file: PathBuf::from(".runtime/update-state.lock"),
update_temp_dir: PathBuf::from(".runtime/update-tmp"),
update_state_file: PathBuf::from("update-state.json"),
update_lock_file: PathBuf::from("update.lock"),
update_temp_dir: PathBuf::from("update-tmp"),
..SystemUpdateConfig::default()
};
HostConfig {
@@ -58,6 +92,7 @@ fn valid_config() -> HostConfig {
host: "127.0.0.1".to_owned(),
port: 8080,
},
runtime_data_dir: PathBuf::from("runtime-data"),
logging: LoggingConfig {
level: "info".to_owned(),
stdout: true,
+128 -1
View File
@@ -1,8 +1,21 @@
use std::path::PathBuf;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use gateway_host::config::{FileLoggingConfig, HostConfig, ListenConfig, LoggingConfig};
use gateway_host::system_update::SystemUpdateConfig;
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
#[test]
fn logging_requires_at_least_one_sink() {
let mut config = HostConfig {
@@ -10,6 +23,7 @@ fn logging_requires_at_least_one_sink() {
host: "127.0.0.1".to_owned(),
port: 8080,
},
runtime_data_dir: PathBuf::from("/tmp/runtime-data"),
logging: LoggingConfig {
level: "info".to_owned(),
stdout: false,
@@ -32,3 +46,116 @@ fn logging_requires_at_least_one_sink() {
.is_err()
);
}
#[test]
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
if env::var_os(CHILD_PROCESS_ENV).is_some() {
write_oauth_recovery_log(PathBuf::from(
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
));
return;
}
let directory = tempfile::tempdir().expect("create log directory");
let output = Command::new(env::current_exe().expect("current test executable"))
.args([
"--exact",
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
"--nocapture",
])
.env(CHILD_PROCESS_ENV, "1")
.env(LOG_DIRECTORY_ENV, directory.path())
.env(
"RUST_LOG",
"off,logging_test_application=info,oauth_recovery=off",
)
.output()
.expect("run logging child process");
assert!(
output.status.success(),
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
let application_log = read_log_file_set(directory.path(), APPLICATION_LOG_FILE_PREFIX);
assert!(application_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
assert!(application_log.contains(APPLICATION_LOG_MARKER));
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
assert!(recovery_log.contains(OAUTH_RECOVERY_LOG_MARKER));
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
}
fn write_oauth_recovery_log(directory: PathBuf) {
let config = HostConfig {
listen: ListenConfig {
host: "127.0.0.1".to_owned(),
port: 8080,
},
runtime_data_dir: PathBuf::from("/tmp/runtime-data"),
logging: LoggingConfig {
level: "off".to_owned(),
stdout: false,
file: FileLoggingConfig {
enabled: true,
directory,
retention_days: 1,
max_file_size_mb: 1,
max_files: 1,
},
},
system_update: SystemUpdateConfig::default(),
drain_timeout_seconds: 30,
worker_shutdown_timeout_seconds: 30,
};
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("create logging runtime");
let bundle = runtime
.block_on(gateway_host::initialize(config))
.expect("initialize logging");
tracing::info!(
target: APPLICATION_LOG_TARGET,
marker = APPLICATION_LOG_MARKER,
"application file test record"
);
tracing::info!(
target: OAUTH_RECOVERY_LOG_TARGET,
provider = OAUTH_RECOVERY_PROVIDER,
marker = OAUTH_RECOVERY_LOG_MARKER,
"OAuth recovery test record"
);
drop(bundle);
}
fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
let mut files = fs::read_dir(directory)
.expect("read log directory")
.map(|entry| entry.expect("read log entry").path())
.filter(|path| {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with(file_prefix))
})
.collect::<Vec<_>>();
files.sort();
assert!(!files.is_empty(), "expected log file set {file_prefix}");
files
.into_iter()
.map(|path| fs::read_to_string(path).expect("read log file"))
.collect()
}
fn json_target_field(target: &str) -> String {
format!(r#""target":"{target}""#)
}
@@ -208,11 +208,24 @@ pub struct RateLimitWindow {
pub reset_at: Option<i64>,
}
/// 限流项键的来源。
///
/// HTTP headers 和事件里的 `metered_limit_name` 提供稳定 `limit_id`;
/// WebSocket `additional_rate_limits` 在部分协议版本里只以可读名称为 map key,
/// 此时必须由持久化层用既有名称元数据解析,不能把名称误当成稳定 ID。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RateLimitKeySource {
LimitId,
LimitName,
}
/// 单个计量项的限流信息。
#[derive(Debug, Clone, PartialEq)]
pub struct RateLimitDetails {
/// 稳定的计量项 ID。
pub limit_id: String,
/// [`limit_id`](Self::limit_id) 是稳定 ID,还是名称生成的临时解析键。
pub key_source: RateLimitKeySource,
/// 上游提供的可读名称。
pub limit_name: Option<String>,
/// 当前请求是否被允许。
@@ -315,26 +328,66 @@ pub fn is_rate_limit_header_name(name: &str) -> bool {
|| rate_limit_id_from_header_name(&normalized).is_some()
}
/// 判断响应头是否为仅供代理本地观测的 Codex 额度信号。
///
/// 这类头会让下游官方 Codex 客户端显示账户额度提示,因此不能穿过账号隔离边界。
/// 通用限流头(例如 `retry-after`)和非额度的 Codex 头仍允许下发。
pub fn is_codex_quota_header_name(name: &str) -> bool {
let normalized = name.trim().to_ascii_lowercase();
matches!(
normalized.as_str(),
"x-codex-credits-has-credits"
| "x-codex-credits-unlimited"
| "x-codex-credits-balance"
| "x-codex-active-limit"
| "x-codex-plan-type"
| "x-codex-promo-message"
| "x-codex-rate-limit-reached-type"
| "x-codex-primary-over-secondary-limit-percent"
) || rate_limit_id_from_header_name(&normalized)
.is_some_and(|limit_id| limit_id == "codex" || limit_id.starts_with("codex_"))
}
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
if value.get("type").and_then(Value::as_str) != Some("codex.rate_limits") {
return None;
}
let limit_id = rate_limit_name(value)
.map(normalize_limit_id)
let explicit_limit_id = top_level_rate_limit_id(value).map(normalize_limit_id);
let explicit_limit_name = non_empty_string(value, "limit_name");
let default_limit_id = explicit_limit_id
.clone()
.unwrap_or_else(|| "codex".to_string());
let limit_name = value
.get("metered_limit_name")
.and_then(Value::as_str)
.and_then(|_| value.get("limit_name"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|name| !name.is_empty())
.map(ToString::to_string);
let details = value
.get("rate_limits")
.and_then(|details| parse_details_from_object(details, &limit_id, limit_name));
let default_details = value.get("rate_limits").and_then(|details| {
parse_details_from_object(
details,
&default_limit_id,
explicit_limit_name,
RateLimitKeySource::LimitId,
)
});
let mut limits = parse_additional_rate_limits(value);
let matching_additional_ids = default_details
.as_ref()
.map(|default| {
limits
.iter()
.filter(|(_, additional)| same_rate_limit_facts(default, additional))
.map(|(limit_id, _)| limit_id.clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
// 当前 WebSocket 协议会把活动具名桶同时镜像到顶层 `rate_limits`,但不附
// `metered_limit_name`。只要 additional 中存在相同事实,顶层就不是 core 桶。
let default_is_additional_alias =
explicit_limit_id.is_none() && !matching_additional_ids.is_empty();
if let Some(details) = default_details
&& !default_is_additional_alias
{
limits.insert(default_limit_id.clone(), details);
}
let credits = value.get("credits").and_then(parse_credits_from_object);
let plan_type = value
.get("plan_type")
@@ -343,17 +396,18 @@ pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
.filter(|plan| !plan.is_empty())
.map(ToString::to_string);
if details.is_none() && credits.is_none() && plan_type.is_none() {
if limits.is_empty() && credits.is_none() && plan_type.is_none() {
return None;
}
let mut limits = BTreeMap::new();
if let Some(details) = details {
limits.insert(limit_id.clone(), details);
}
let active_limit = explicit_limit_id.or_else(|| match matching_additional_ids.as_slice() {
[limit_id] => Some(limit_id.clone()),
[] if !limits.is_empty() => Some(default_limit_id),
_ => None,
});
Some(ParsedRateLimits {
limits,
active_limit: Some(limit_id),
active_limit,
credits,
plan_type,
promo_message: value
@@ -577,6 +631,7 @@ fn parse_details_from_lookup(
}
Some(RateLimitDetails {
limit_id: limit_id.to_string(),
key_source: RateLimitKeySource::LimitId,
limit_name,
allowed,
limit_reached,
@@ -667,6 +722,7 @@ fn parse_details_from_object(
value: &Value,
limit_id: &str,
limit_name: Option<String>,
key_source: RateLimitKeySource,
) -> Option<RateLimitDetails> {
let primary = value.get("primary").and_then(parse_window_from_object);
let secondary = value.get("secondary").and_then(parse_window_from_object);
@@ -677,6 +733,7 @@ fn parse_details_from_object(
}
Some(RateLimitDetails {
limit_id: limit_id.to_string(),
key_source,
limit_name,
allowed,
limit_reached,
@@ -732,11 +789,59 @@ fn value_as_positive_i64(value: &Value) -> Option<i64> {
.filter(|value| *value > 0)
}
fn rate_limit_name(value: &Value) -> Option<&str> {
fn rate_limit_id(value: &Value) -> Option<&str> {
value
.get("metered_limit_name")
.or_else(|| value.get("limit_name"))
.or_else(|| value.get("limit_id"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn top_level_rate_limit_id(value: &Value) -> Option<&str> {
rate_limit_id(value).or_else(|| {
value
.get("limit_name")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
})
}
fn non_empty_string(value: &Value, field: &str) -> Option<String> {
value
.get(field)
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
fn parse_additional_rate_limits(value: &Value) -> BTreeMap<String, RateLimitDetails> {
value
.get("additional_rate_limits")
.and_then(Value::as_object)
.into_iter()
.flatten()
.filter_map(|(map_key, value)| {
let explicit_limit_id = rate_limit_id(value).map(normalize_limit_id);
let (limit_id, key_source) = explicit_limit_id.map_or_else(
|| (normalize_limit_id(map_key), RateLimitKeySource::LimitName),
|limit_id| (limit_id, RateLimitKeySource::LimitId),
);
let limit_name = non_empty_string(value, "limit_name")
.or_else(|| Some(map_key.trim().to_owned()).filter(|name| !name.is_empty()));
parse_details_from_object(value, &limit_id, limit_name, key_source)
.map(|details| (limit_id, details))
})
.collect()
}
fn same_rate_limit_facts(left: &RateLimitDetails, right: &RateLimitDetails) -> bool {
left.allowed == right.allowed
&& left.limit_reached == right.limit_reached
&& left.primary == right.primary
&& left.secondary == right.secondary
}
fn normalize_limit_id(value: impl AsRef<str>) -> String {
@@ -70,7 +70,7 @@ pub const MAX_SSE_EVENT_BUFFER_BYTES: usize = 64 * 1024 * 1024;
pub const DONE_SSE_FRAME: &str = "data: [DONE]\n\n";
/// 用于处理任意分块边界的增量 SSE 解码器。
#[derive(Debug, Default)]
#[derive(Debug)]
pub struct SseEventDecoder {
pending: Vec<u8>,
/// `pending` 中已确认不含帧分隔符的前缀长度。新数据到达时从该边界
@@ -79,6 +79,19 @@ pub struct SseEventDecoder {
/// 超大未完成帧已开始直接交付;在遇到帧分隔符前只保留跨 chunk 扫描尾部,
/// 不再尝试解析该帧。
opaque_frame: bool,
/// 只有整个 SSE 流的第一条物理行允许携带 UTF-8 BOM。
stream_start: bool,
}
impl Default for SseEventDecoder {
fn default() -> Self {
Self {
pending: Vec::new(),
scanned: 0,
opaque_frame: false,
stream_start: true,
}
}
}
impl SseEventDecoder {
@@ -115,7 +128,8 @@ impl SseEventDecoder {
while let Some(end) = self.next_frame_end(consumed) {
let frame = std::str::from_utf8(&self.pending[consumed..end])
.map_err(|error| SseError::ParseError(error.to_string()))?;
events.extend(parse_sse_events(frame)?);
events.extend(parse_sse_events_inner(frame, self.stream_start)?);
self.stream_start = false;
consumed = end;
}
@@ -149,9 +163,10 @@ impl SseEventDecoder {
} else {
std::str::from_utf8(&raw)
.ok()
.and_then(|frame| parse_sse_events(frame).ok())
.and_then(|frame| parse_sse_events_inner(frame, self.stream_start).ok())
.unwrap_or_default()
};
self.stream_start = false;
frames.push(SseFrame::new(raw, events));
consumed = end;
}
@@ -173,7 +188,9 @@ impl SseEventDecoder {
let pending = std::mem::take(&mut self.pending);
let frame = std::str::from_utf8(&pending)
.map_err(|error| SseError::ParseError(error.to_string()))?;
parse_sse_events(frame)
let events = parse_sse_events_inner(frame, self.stream_start)?;
self.stream_start = false;
Ok(events)
}
/// 在流结束时返回未带空行分隔符的最后一帧及其旁路解析结果。
@@ -191,9 +208,10 @@ impl SseEventDecoder {
} else {
std::str::from_utf8(&raw)
.ok()
.and_then(|frame| parse_sse_events(frame).ok())
.and_then(|frame| parse_sse_events_inner(frame, self.stream_start).ok())
.unwrap_or_default()
};
self.stream_start = false;
self.opaque_frame = false;
vec![SseFrame::new(raw, events)]
}
@@ -207,6 +225,7 @@ impl SseEventDecoder {
let remaining = self.pending.split_off(end);
let raw = std::mem::replace(&mut self.pending, remaining);
frames.push(SseFrame::new(raw, Vec::new()));
self.stream_start = false;
self.opaque_frame = false;
self.scanned = 0;
true
@@ -226,6 +245,7 @@ impl SseEventDecoder {
let raw = std::mem::replace(&mut self.pending, tail);
self.scanned = 0;
frames.push(SseFrame::new(raw, Vec::new()));
self.stream_start = false;
}
}
@@ -351,12 +371,21 @@ impl EventBuilder {
/// 解析 SSE 事件流。
pub fn parse_sse_events(input: &str) -> Result<Vec<SseEvent>, SseError> {
parse_sse_events_inner(input, true)
}
fn parse_sse_events_inner(input: &str, strip_leading_bom: bool) -> Result<Vec<SseEvent>, SseError> {
let mut events = Vec::new();
let mut builder = EventBuilder::default();
let mut saw_sse_syntax = false;
let mut event_buffer_bytes = 0usize;
for raw_line in input.lines() {
for (index, raw_line) in input.lines().enumerate() {
let raw_line = if strip_leading_bom && index == 0 {
raw_line.strip_prefix('\u{feff}').unwrap_or(raw_line)
} else {
raw_line
};
let line = raw_line.strip_suffix('\r').unwrap_or(raw_line);
if line.is_empty() {
event_buffer_bytes = 0;

Some files were not shown because too many files have changed in this diff Show More