Compare commits

...
74 Commits
Author SHA1 Message Date
ZyyandCodex 95f6434443 chore: bump version to 3.1.2
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 13:00:45 +08:00
ZyyandZyy 8734b0d3d5 fix(openai): keep fingerprint-pinned TLS versions (#35)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-08-03 11:59:59 +08:00
dependabot[bot] 790d0d9628 chore(deps): bump the minor-and-patch group in /backend with 3 updates (#32)
Bumps the minor-and-patch group in /backend with 3 updates: [redis](https://github.com/redis-rs/redis-rs), [hyper-rustls](https://github.com/rustls/hyper-rustls) and [rustls](https://github.com/rustls/rustls).


Updates `redis` from 1.4.1 to 1.5.0
- [Release notes](https://github.com/redis-rs/redis-rs/releases)
- [Commits](https://github.com/redis-rs/redis-rs/compare/redis-1.4.1...redis-1.5.0)

Updates `hyper-rustls` from 0.27.7 to 0.27.9
- [Release notes](https://github.com/rustls/hyper-rustls/releases)
- [Commits](https://github.com/rustls/hyper-rustls/compare/v/0.27.7...v/0.27.9)

Updates `rustls` from 0.23.36 to 0.23.43
- [Release notes](https://github.com/rustls/rustls/releases)
- [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rustls/rustls/compare/v/0.23.36...v/0.23.43)

---
updated-dependencies:
- dependency-name: redis
  dependency-version: 1.5.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: hyper-rustls
  dependency-version: 0.27.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: rustls
  dependency-version: 0.23.43
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-03 11:39:06 +08:00
dependabot[bot]andZyy 3c562563c5 chore(deps): bump p256 from 0.13.2 to 0.14.0 in /backend (#33)
* chore(deps): bump p256 from 0.13.2 to 0.14.0 in /backend

Bumps [p256](https://github.com/RustCrypto/elliptic-curves) from 0.13.2 to 0.14.0.
- [Commits](https://github.com/RustCrypto/elliptic-curves/compare/p256/v0.13.2...p256/v0.14.0)

---
updated-dependencies:
- dependency-name: p256
  dependency-version: 0.14.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix(xai): update p256 point encoding API

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-08-03 11:38:22 +08:00
dependabot[bot] f19785d0e1 chore(deps): bump the minor-and-patch group in /frontend with 6 updates (#34)
Bumps the minor-and-patch group in /frontend with 6 updates:

| Package | From | To |
| --- | --- | --- |
| [@lucide/vue](https://github.com/lucide-icons/lucide/tree/HEAD/packages/vue) | `1.27.0` | `1.28.0` |
| [@vueuse/core](https://github.com/vueuse/vueuse/tree/HEAD/packages/core) | `14.3.0` | `14.4.0` |
| [axios](https://github.com/axios/axios) | `1.18.1` | `1.19.0` |
| [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) | `26.1.1` | `26.1.2` |
| [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) | `8.1.5` | `8.2.0` |
| [vue-tsc](https://github.com/vuejs/language-tools/tree/HEAD/packages/tsc) | `3.3.8` | `3.3.9` |


Updates `@lucide/vue` from 1.27.0 to 1.28.0
- [Release notes](https://github.com/lucide-icons/lucide/releases)
- [Commits](https://github.com/lucide-icons/lucide/commits/1.28.0/packages/vue)

Updates `@vueuse/core` from 14.3.0 to 14.4.0
- [Release notes](https://github.com/vueuse/vueuse/releases)
- [Commits](https://github.com/vueuse/vueuse/commits/v14.4.0/packages/core)

Updates `axios` from 1.18.1 to 1.19.0
- [Release notes](https://github.com/axios/axios/releases)
- [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md)
- [Commits](https://github.com/axios/axios/compare/v1.18.1...v1.19.0)

Updates `@types/node` from 26.1.1 to 26.1.2
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

Updates `vite` from 8.1.5 to 8.2.0
- [Release notes](https://github.com/vitejs/vite/releases)
- [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md)
- [Commits](https://github.com/vitejs/vite/commits/create-vite@8.2.0/packages/vite)

Updates `vue-tsc` from 3.3.8 to 3.3.9
- [Release notes](https://github.com/vuejs/language-tools/releases)
- [Changelog](https://github.com/vuejs/language-tools/blob/master/CHANGELOG.md)
- [Commits](https://github.com/vuejs/language-tools/commits/v3.3.9/packages/tsc)

---
updated-dependencies:
- dependency-name: "@lucide/vue"
  dependency-version: 1.28.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: "@vueuse/core"
  dependency-version: 14.4.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: axios
  dependency-version: 1.19.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: "@types/node"
  dependency-version: 26.1.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: vite
  dependency-version: 8.2.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
- dependency-name: vue-tsc
  dependency-version: 3.3.9
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-03 11:37:31 +08:00
dependabot[bot] a3d098a87c chore(deps): bump the minor-and-patch group with 3 updates (#31)
Bumps the minor-and-patch group with 3 updates: [github/codeql-action/upload-sarif](https://github.com/github/codeql-action), [actions/attest](https://github.com/actions/attest) and [docker/login-action](https://github.com/docker/login-action).


Updates `github/codeql-action/upload-sarif` from 4.37.3 to 4.37.4
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81...f205ea1c3313d32999d8d6a48b4f6530d4437b38)

Updates `actions/attest` from 4.2.0 to 4.2.1
- [Release notes](https://github.com/actions/attest/releases)
- [Changelog](https://github.com/actions/attest/blob/main/RELEASE.md)
- [Commits](https://github.com/actions/attest/compare/f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6...508db95dd578ae2727ebd6217d5ba78e4fbda05d)

Updates `docker/login-action` from 4.5.1 to 4.6.0
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](https://github.com/docker/login-action/compare/abd2ef45e78c5afb21d64d4ca52ee8550d9572c7...dbcb813823bdd20940b903addbd779551569679f)

---
updated-dependencies:
- dependency-name: github/codeql-action/upload-sarif
  dependency-version: 4.37.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: actions/attest
  dependency-version: 4.2.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: minor-and-patch
- dependency-name: docker/login-action
  dependency-version: 4.6.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: minor-and-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-03 11:37:12 +08:00
ZyyandClaude Fable 5 449d7c4218 refactor(openai): inline single-call recovery confirmation helper
quota_recovery_confirmed had one call site; inlining the evidence match
into quota_success_state makes the recovery rules (successful response
always recovers; refreshed snapshot requires old reset elapsed) visible
at the decision point. Rust best practices §1.8: don't extract helpers
for single-call simple logic.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-03 03:36:17 +08:00
ZyyandClaude Fable 5 d199141a91 feat(store): materialize quota_limit_reached for single-source limit status
Dashboard reads the materialized column instead of parsing raw quota
JSON (design §3.3/§5.3): Provider computes snapshot-level limit_reached
on quota writes, store persists it, Admin account_status and Dashboard
both consume the same value.

- QuotaObservation gains limit_reached; openai writes fact.exhausted()
  (429 always true), xai writes used_percent>=100
- compare_and_swap_provider_quota updates the column; get_quotas and
  ACCOUNT_SELECT read it back
- ProviderAccountSummary carries quota_limit_reached; admin_account_status
  derives RateLimited from it (Unknown -> Invalid)
- Dashboard provider_account_metrics reads quota_limit_reached instead of
  provider_quota_json->'rate_limit'
- Migration 0003 adds the column with backfill (coalesce for NULL when
  rate_limit key is absent); 0004 fixes NULL backfill for already-applied
  environments

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-03 03:32:17 +08:00
ZyyandClaude Fable 5 df83ffdafb refactor(quota): split quota module and align scheduling/display semantics
Split credential/quota.rs into quota/{mod,snapshot,recovery}.rs with
corresponding test modules. Snapshot aggregation is snapshot-level:
top-level limit_reached/allowed and any window used>=100 count as
exhaustion, scheduling signals survive limit-only windows, and window
rolling is O(1) arithmetic.

402 recovery uses the pre-write confirmed reset (old reset elapsed plus
fresh snapshot not exhausted, or explicit allowed=true) instead of the
new snapshot's future reset, fixing the worker recovery deadlock.
Scheduling projection TTL is capped at the nearest active reset so
cache expiry never outlives the limit.

Admin derives seven mutually exclusive statuses (Active/RateLimited/
QuotaExhausted/Expired/Invalid/Disabled/Banned) from availability plus
snapshot-level quota.limitReached; Unknown maps to invalid. Dashboard
metrics are mutually exclusive (rate_limited disjoint from active),
ignore reset-expired windows, and cover additional rate limits.

xAI: bare 402 writes a short account cooldown instead of persisting
QuotaExhausted; ModelQuotaExhausted keeps model scope; runtime cooldown
survives credential rotation; account deletion clears all account and
model-scoped cooldown keys via new ProviderCooldownPort::clear_all.

Migration 0002 is frozen in the sha256 manifest.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-03 03:04:43 +08:00
ZyyandClaude Fable 5 cf03c56cfa refactor(quota): separate rate limiting from account availability
Remove the persistent Cooldown state and cooldown_until/availability_reason
columns; rate limiting (429/503) now lives entirely in provider quota data
with projected window rolling, while availability carries only terminal or
intervention-required states (QuotaExhausted for 402, worker auto-recovers
on quota reset). xAI migrated to the same model (runtime cooldown via Redis,
no persistent state). Frontend derives 限流中 from quota window limitReached.

Align quota window parsing with the official codex schema: drop the invented
monthly_limit/spend_control windows, add the top-level code_review_rate_limit
window, prefer upstream limit_name for display labels, and keep duration
guessing only for grouping.

Also fix: account directory degrades one failing quota to an empty projection
instead of failing the whole page; dashboard rate_limited bucket SQL; fake
store Mutex re-entrancy deadlock; remove dead tokenRefreshing field.

Migration 0002 drops cooldown columns and rewrites the availability CHECK.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-03 00:13:56 +08:00
ZyyandCodex 3d1e33558a style(openai): fix formatting in websocket pump context changes
Co-authored-by: Codex <noreply@openai.com>
2026-08-02 18:27:42 +08:00
ZyyandCodex f314ebb3a2 fix(openai): align wire profile os_type and harden quota 5xx backoff
- Default wire profile os_type to macOS to match the audited official
  desktop user-agent
- Retry quota usage 5xx rejections with 1s/2s exponential backoff,
  leaving 4xx to the existing quota state transitions
- Classify 503 with Retry-After into a resettable cooldown instead of
  an unclassified transient warning
- Add account_id and conversation hash to WebSocket pump exit logs so
  idle-connection resets can be attributed to an account

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 14:27:04 +08:00
ZyyandCodex 31d6ef9585 chore: bump version to 3.1.1
Co-authored-by: Codex <noreply@openai.com>
2026-08-02 04:29:44 +08:00
ZyyandCodex fdfb77c4a2 ci(release): reuse caches across versions
Dispatch releases from the long-lived publishing branch so Cargo caches are shared across version tags. Store prebuilt image caches in per-architecture GHCR refs and include toolchain and workspace manifests in Cargo cache invalidation.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 04:27:49 +08:00
ZyyandCodex a93e3853f7 chore: consolidate migrations and tune settings colors
Fold the completed provider snapshot, S3 backup, and redundant-index changes into the initial PostgreSQL baseline. Keep the settings section tabs aligned with the input surface color.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 04:07:37 +08:00
zyycnandCodex 34da9d7cc8 chore: bump version to 3.1.0
Co-authored-by: Codex <noreply@openai.com>
2026-08-02 03:05:16 +08:00
zyycnandCodex b3dd6bbe46 feat(openai): transparently retry websocket connection limit errors
When upstream rejects a NewChain request with websocket_connection_limit_reached
before the first deliverable event, retry once on a fresh connection and
downgrade to HTTP/SSE if the new connection is still limited or the retry
preparation fails, matching the official retry semantics. Non-NewChain
requests keep the previous behavior of delivering the error frame as-is,
and the code is persisted in upstream diagnostics. All pre-delivery
fallbacks share one helper that marks the transport decision and wait
metrics.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 03:04:06 +08:00
zyycnandCodex 9ab3e45288 feat(openai): zstd-compress responses request bodies like the official client
The official Codex app-server enables enable_request_compression by
default (Stable, default_enabled=true) and sends /v1/responses bodies as
zstd level 3 with Content-Encoding: zstd. Serialize the upstream request
to JSON, compress with zstd level 3, and attach the encoding header on
the HTTP/SSE path; WebSocket continues to use permessage-deflate.

Adds RequestBodyEncode/RequestCompression error variants, a zstd capture
helper for wiremock tests, and a transport test asserting the compressed
body decodes back to the original JSON.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 02:27:46 +08:00
zyycnandCodex d255a86644 test(gateway): sync config rejection fixtures to current example
The example config moved to codex_version 0.146.0 and desktop_build 6119,
but the rejection tests still replaced the old values, silently no-op'ing
and accepting invalid configs.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 02:20:29 +08:00
zyycnandCodex f3f50a818f fix(openai): install ring crypto provider before rustls use
rustls 0.23 compiles both ring and aws-lc-rs in this workspace (aws-sdk-s3
pulls aws-lc-rs), so the process-level CryptoProvider cannot be auto-selected
and any TLS use panics inside spawned tasks - which made the websocket idle
timeout test hang forever. Install the pinned ring provider once before the
first TLS use, with a startup call in the gateway binary as a fallback.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 02:20:28 +08:00
zyycnandCodex 1ff520659f fix(openai,xai): match official accept-encoding and block attestation passthrough
Official Codex Desktop app-server and Grok CLI build reqwest without
compression features, so upstream requests carry no Accept-Encoding.
Drop the gzip/brotli/zstd/deflate features and the explicit
'accept-encoding: identity' Grok header so the wire matches official
clients. Verified: reqwest only inserts Accept-Encoding through the
tower-http Decompression layer, which is compiled solely when a
compression feature is enabled; the official binary's 'gzip, deflate'
string is the h2 HPACK static table entry, not a request header.

Also stop downstream clients from injecting device attestation/integrity
headers (x-oai-attestation, x-oai-is, x-oai-is-update) at both the API
passthrough boundary and the provider header builder; only the official
client or the gateway itself may supply those.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 02:13:08 +08:00
zyycnandCodex 33381fc472 fix(openai): pin TLS stack to official Codex Desktop fingerprint
Pin rustls =0.23.36 and hyper-rustls =0.27.7 so the upstream ClientHello
matches the official app-server binary (reqwest 0.12.28 + ring unchanged).
Verified against ChatGPT.app Resources/codex from the current 26.727.51351
build: rustls 0.23.36, hyper-rustls 0.27.7, ring 0.17.14, reqwest 0.12.28.

rustls 0.23.37+ reorders ClientHello extensions, which changes JA3 even
though cipher suites, groups, signature algorithms and ALPN stay identical.
Keep the ring provider: reqwest rustls-tls-native-roots is __rustls-ring
and matches the official binary; do not switch the TLS path to aws-lc-rs
without re-capturing the handshake.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:59:58 +08:00
zyycnandCodex e3d767bf54 fix(frontend): fill brand mark box with favicon tile
The sidebar and login logo boxes kept a #1c2a3d background around a
smaller favicon image, leaving an inconsistent ring in dark mode. The
image now fills the box and the divergent box background is removed, so
the favicon's own black tile is the visual mark.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:53:05 +08:00
zyycnandCodex 8f82d0e100 chore(config): refresh wire profile baselines to current official releases
OpenAI: Codex Core 0.146.0, Desktop 26.727.51351 build 6119, and os_type
macOS (official Rust app-server os_info output) so the startup User-Agent
matches the official client before the periodic release check catches up.
xAI: grok-shell client_version 0.2.118, the current @xai-official/grok
npm release.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:46:31 +08:00
zyycnandCodex 3085e5aabe style(core): apply rustfmt to decimal canonical test
The accounting delegate commit left this test unformatted, which broke
`cargo fmt --check` in CI.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:35:23 +08:00
zyycnandCodex 36dce639a9 test: freeze settings contract fixture and exact workspace DAG
Adds a full-field RuntimeSettings wire fixture with request/response field
lockstep, and a cargo-metadata based guard that compares every workspace
internal runtime edge against a frozen allowed-edges table.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:32:49 +08:00
zyycnandCodex 6e15757150 refactor(api): stop duplicating core fields into usage metadata
UsageRecord metadata now carries only provider observations; protocol,
latency details, http version, status codes, websocket pool and image
generation facts moved to the top-level record contract. Contract tests
were updated to pin the new wire shape.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:25:57 +08:00
zyycnandCodex 365d389a14 refactor(frontend): phase system-update state and clear remaining any
system-update now uses a discriminated phase union so mutually exclusive
operation states cannot be expressed together; connection-test event shapes
and cost chart points are typed instead of any.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:25:53 +08:00
zyycnandCodex e79d57eaf9 refactor(frontend): normalize usage records into a single view model
Components now consume `UsageViewModel` produced by `normalizeUsageRecord`;
metadata fallback exists only in the normalizer. Also complete the typed
request params for ops errors so the usage module stays build-clean.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:25:50 +08:00
zyycnandCodex 8d5dfea6de fix(openai,api): unify upstream fingerprint to official Codex Desktop profile
Block client originator/User-Agent at both the API passthrough boundary
and the provider header builder, so every upstream request always carries
the runtime wire profile instead of leaking downstream client fingerprints.

Generate User-Agent in the official Codex Desktop Rust-core format
(originator/version (os; arch) terminal) and drop the fabricated
'(Codex Desktop; <desktop version>)' suffix. Verified against the current
official client 26.727.51351, which spawns the app-server with
CODEX_INTERNAL_ORIGINATOR_OVERRIDE=Codex Desktop and never appends a
desktop version to the model-request User-Agent.

WebSocket pool fingerprint tests now rotate on the Codex Core release
instead of the desktop release, matching the fields that actually change
the User-Agent.

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:24:06 +08:00
zyycnandCodex e02e5b186e refactor(store,api): split hotspot modules per boundary audit
Split the four oversized files into mod.rs-style module directories without
changing public ports or behavior:

- gateway-store: observability/{model,mapping,admin_adapter,queries/*},
  provider_accounts/{rows,repository,mapping,admin_adapter,core_adapter},
  root {config,bundle,workers,admin_adapter,value}
- gateway-api: admin/observability/{query,wire,routes,presenter} with
  mirrored contract tests and updated frozen architecture manifest

Co-authored-by: Codex <noreply@openai.com>
2026-08-02 01:00:42 +08:00
zyycnandClaude Fable 5 2dc670875a feat(ui): type system update log events
Replace any[] updateLogs with SystemUpdateEvent (id, level, message, at,
terminal) so the log pipeline is type-checked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:44 +08:00
zyycnandClaude Fable 5 9768921ff7 refactor(accounting): delegate admin decimal ops to core Decimal
Core Decimal gains canonical() and checked_div_u64(); gateway-admin
DecimalAmount now delegates arithmetic to core instead of reimplementing
scaled parsing, removing the duplicate scaled_amount logic.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:43 +08:00
zyycnandClaude Fable 5 db90f0631a refactor(store): make admin api key comments functional
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:43 +08:00
zyycnandClaude Fable 5 8a0a8f9af4 refactor(store): field-level admin api key update
Replace read-full-control-plane-and-copy-all-fields with a targeted
mutation that updates only admin_api_key plus revision and audit. No
longer implicitly coupled to unrelated runtime settings fields.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:43 +08:00
zyycnandClaude Fable 5 8ca2c68e90 refactor(provider): self-contained schemaVersion comments
Drop cross-references to audit docs in code comments.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:43 +08:00
zyycnandClaude Fable 5 d26fc44fe6 feat(provider): version provider observation metadata with schemaVersion
OpenAI and xAI now stamp provider_observation_json with schemaVersion: 1 so
historical records can be decoded per version (missing version = v0). Per
project-redundancy-boundary-audit DB-03.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:43 +08:00
zyycnandClaude Fable 5 7ed2a6668a feat(usage): mark attempt list as best-effort observations
Decide attempt facts are best-effort (adr-002), not authoritative history:
- Usage detail response exposes attemptsComplete: false
- frontend labels the attempt chain as best-effort
- architecture.md documents the semantics

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:31:39 +08:00
zyycnandClaude Fable 5 3c4ac99ae4 feat(api): type request params across admin API modules
Replace data: object with precise parameter types derived from call sites,
using type inference (Omit/union) over bulk interface declarations where
possible. Covers accounts, api-keys, usage, system, auth, dashboard,
settings.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:08:52 +08:00
zyycnandClaude Fable 5 63d276767e refactor(api): dedupe admin presenter helpers and page metadata
- extract format_number/format_compact_number into admin/presenter.rs
- use gateway_admin model china_day_start instead of local copy
- unify PageMeta onto admin::wire, drop observability duplicate
- accounts reuse map_admin_service_error instead of local match

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:02:23 +08:00
zyycnandClaude Fable 5 60dcc5b0db feat(db): drop redundant provider_accounts_id_kind_uq index
The (id, provider_kind) unique index adds no constraint since id is already
the primary key and nothing references the composite key. New forward
migration 0004 per project-redundancy-boundary-audit DB-01; frozen 0001
untouched.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 00:02:19 +08:00
zyycnandCodex 0d2c648422 fix(deps): resolve RustSec dependency advisories
- bump event-listener to 5.4.2 for RUSTSEC-2026-0221
- use the AWS SDK modern HTTPS client to remove rustls-webpki 0.101 affected by RUSTSEC-2026-0098, RUSTSEC-2026-0099, and RUSTSEC-2026-0104

Co-authored-by: Codex <noreply@openai.com>
2026-08-01 23:53:55 +08:00
zyycnandClaude Fable 5 56f7ddd1dc test: mirror test directory structure to production modules
Reorganize integration tests so tests/ mirrors src/ per architecture rule.

- gateway-admin: split flat backup tests into tests/backup/{policy,task,support}
  and tests/model/backup.rs, sharing fakes via backup::support
- gateway-store: move coordination_buffer and execution_buffer tests into
  tests/redis/ and tests/postgres/ to match src layout

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 23:28:27 +08:00
zyycnandClaude Fable 5 930e311f51 docs(backup): document S3 backup API and architecture
- api.md: backup endpoints under /api/admin/settings/backups/*
- architecture.md: backup service wiring notes

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 22:13:59 +08:00
zyycnandClaude Fable 5 5df8c71b89 chore: ignore local .claude directory
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 22:12:27 +08:00
zyycnandClaude Fable 5 dfedb5069d feat(backup): add S3-compatible PostgreSQL backups with expiry TTL
Implement end-to-end backup to S3-compatible object storage (Cloudflare R2):
admin settings UI, scheduled and manual jobs, records, retention and expiry,
download via presigned URL, and delete convergence.

- gateway-admin: backup model, schedule/retention policy, daemon worker,
  storage identity lock and audit
- gateway-store: pg_dump staging, S3 client with probe, backup repository
- gateway-api: backup settings/records/create/download/delete routes
- migrations: fold expiry TTL into 0003_s3_backup.sql (single migration)
- frontend: S3 storage, schedule and records cards under settings/backup

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 22:12:23 +08:00
zyycnandCodex 7016c8f401 chore(docs): remove outdated provider account snapshot doc
Co-authored-by: Codex <noreply@openai.com>
2026-08-01 01:16:58 +08:00
zyycnandCodex 5a14b74ff2 feat(frontend): restyle favicon with vintage film grain and mist
- keep black base and white mark unchanged
- add sparse white film-grain noise and radial white mist
- add short, straight, battle-worn scratch marks with rough edges

Co-authored-by: Codex <noreply@openai.com>
2026-08-01 01:16:58 +08:00
zyycnandCodex 00bab8d390 chore: bump version to 3.0.33
Co-authored-by: Codex <noreply@openai.com>
2026-08-01 00:49:05 +08:00
zyycnandCodex f6e364ca33 feat(observability): surface affinity, quota and catalog failure details
- carry underlying redis/serialization error into native continuation logs
- attach provider/account to affinity write failures and timeouts
- log quota cycle summary counts and per-account transient causes
- include upstream status in catalog/quota error diagnostics
- record request_id/method/uri in HTTP trace spans for failed responses

Co-authored-by: Codex <noreply@openai.com>
2026-08-01 00:42:33 +08:00
zyycnandCodex 4cd98889a7 refactor(core): centralize rotation strategy and circuit policy
Own RotationStrategy wire mapping in gateway-core and re-export it from gateway-admin, move ProviderCircuitPolicy into the execution port contract, and rename the Core coordination timeout without a Redis implementation leak.

Co-authored-by: Codex <noreply@openai.com>
2026-07-31 23:23:23 +08:00
zyycnandCodex c9b613659c fix(openai): refresh GPT pricing from upstream
Align GPT-5.6 Terra/Luna standard, Fast, cache and long-context rates with the current upstream pricing page, and add GPT-4o/4o-mini cached-input rates.

Co-authored-by: Codex <noreply@openai.com>
2026-07-31 22:43:18 +08:00
zyycnandCodex 01b4029998 feat(observability): polish usage detail and error diagnostics
Show attempt chains and P95 latency, surface client status/component/endpoint, and remove unused dashboard summary fields.

Co-authored-by: Codex <noreply@openai.com>
2026-07-31 22:28:24 +08:00
zyycnandCodex f583e50e88 feat(observability): snapshot provider account identity for history
Persist provider account name, email, and authentication kind at write time so request, attempt, and error views survive account deletion; add stable diagnostic keys with display-name resolution (account email/name, API key name); count only errored requests in failure diagnostics.

Co-authored-by: Codex <noreply@openai.com>
2026-07-31 21:25:38 +08:00
Zyy a7ea957843 test(openai): move provider mapping tests to integration suite 2026-07-31 17:37:20 +08:00
ZyyandCodex c92c95a733 chore: bump version to 3.0.32
Co-authored-by: Codex <noreply@openai.com>
2026-07-31 16:27:35 +08:00
ZyyandCodex ae7608440e fix(openai): replay client-owned conversation history
Co-authored-by: Codex <noreply@openai.com>
2026-07-31 16:07:32 +08:00
ZyyandCodex 5c9da8a39d fix(openai): preserve upstream failures during fallback
Co-authored-by: Codex <noreply@openai.com>
2026-07-31 15:33:22 +08:00
zyycnandCodex e012f24ef2 chore: bump version to 3.0.31
Co-authored-by: Codex <noreply@openai.com>
2026-07-31 02:20:58 +08:00
zyycnandCodex 756c9c7158 fix(openai): preserve exhausted quotas until reset
Co-authored-by: Codex <noreply@openai.com>
2026-07-31 02:20:37 +08:00
zyycnandCodex ab4bf75d0e chore: bump version to 3.0.30
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 21:43:11 +08:00
zyycnandCodex 7a9715ed41 chore(db): fold schema updates into initial migration
Collapse the completed response ID, service tier, and quota cooldown changes into the fresh-database baseline. Refresh the migration contract and state consistently that cross-major online upgrades are unsupported.

Co-authored-by: Codex <noreply@openai.com>
2026-07-30 21:42:57 +08:00
zyycnandCodex c353a6159e fix(openai): preserve failover before stream commit
Hold structural response events for a bounded replay grace so a split-frame 429 can rotate accounts without disconnecting the client. Release immediately on semantic output and keep post-commit failures non-replayable.

Co-authored-by: Codex <noreply@openai.com>
2026-07-30 21:42:30 +08:00
zyycnandCodex f5dca4848a fix(openai): harden exhausted quota recovery
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 21:07:53 +08:00
zyycnandCodex 36321a9af4 chore: bump version to 3.0.29
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 04:03:58 +08:00
zyycnandCodex b485b8bd4a fix(openai): reconcile exhausted account quotas
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 04:03:04 +08:00
zyycnandCodex cae2c8656a fix(xai): preserve quota exhaustion state
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 03:10:35 +08:00
zyycnandCodex acf574129f chore: bump version to 3.0.28
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 02:26:35 +08:00
zyycnandCodex 6426e0d3de fix(accounts): align availability summaries
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 02:22:23 +08:00
zyycnandCodex eedc39b33a chore: bump version to 3.0.27
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:55:51 +08:00
zyycnandCodex 9672a2e000 fix(xai): prioritize structured quota signals
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:53:43 +08:00
zyycnandCodex 0fffd2c550 fix(accounts): converge account state and imports
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:23:42 +08:00
ZyyandCodex 5671a57dc8 chore: bump version to 3.0.26
Co-authored-by: Codex <noreply@openai.com>
2026-07-29 19:44:44 +08:00
Zyy 166e8754a2 fix(xai): restore tool replay and billing details 2026-07-29 18:28:22 +08:00
271 changed files with 28573 additions and 15662 deletions
+1 -1
View File
@@ -76,7 +76,7 @@ jobs:
${{ hashFiles('trivy-results.sarif') != '' &&
(github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository) }}
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
with:
sarif_file: trivy-results.sarif
category: runtime-container
+14 -17
View File
@@ -1,9 +1,6 @@
name: Release
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
@@ -15,7 +12,7 @@ permissions:
contents: read
concurrency:
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
group: release-${{ inputs.tag }}
cancel-in-progress: false
env:
@@ -43,14 +40,14 @@ jobs:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
ref: ${{ inputs.tag }}
fetch-depth: 0
- name: Read release metadata
id: version
shell: bash
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
RELEASE_TAG: ${{ inputs.tag }}
run: |
ruby <<'RUBY' >> "$GITHUB_OUTPUT"
require "json"
@@ -188,9 +185,9 @@ jobs:
~/.cargo/registry
~/.cargo/git
backend/target
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
restore-keys: |
cargo-${{ runner.os }}-${{ matrix.target }}-
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
- name: Build binary
shell: bash
@@ -253,9 +250,9 @@ jobs:
~/.cargo/registry
~/.cargo/git
backend/target
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
restore-keys: |
cargo-${{ runner.os }}-${{ matrix.target }}-
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
- name: Build binary
shell: bash
@@ -378,7 +375,7 @@ jobs:
PY
- name: Generate release asset provenance
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
with:
subject-path: dist/release/*
@@ -454,7 +451,7 @@ jobs:
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to GHCR
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -471,8 +468,8 @@ jobs:
push: true
build-args: |
CPR_PREBUILT_DIR=dist/docker
cache-from: type=gha,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
provenance: mode=max
sbom: true
labels: |
@@ -525,7 +522,7 @@ jobs:
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Login to GHCR
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -584,7 +581,7 @@ jobs:
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
- name: Generate container provenance attestation
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
subject-digest: ${{ steps.manifest.outputs.digest }}
@@ -626,7 +623,7 @@ jobs:
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
body: |
> [!WARNING]
> **Upgrading across major versions?** Database schema and configuration are only compatible within a major version, and the one-click self-update refuses cross-major targets (releases before 3.0 lack this gate — do not use their built-in update to reach a newer major). Deploy the new major fresh (new `.runtime/` data directory) and re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
> **Upgrading across major versions?** Online upgrades across major versions are not supported. Deploy the new major fresh with a new `.runtime/` data directory, then re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
## 📥 Installation
+1
View File
@@ -35,6 +35,7 @@ Thumbs.db
/.codex-ws-audit/
/.agents/
/.reasonix/
/.claude
/.worktrees/
/worktrees/
/.cargo/
+6 -7
View File
@@ -168,10 +168,8 @@ docker compose -f deploy/compose.yaml up -d
> 运行状态。
> [!WARNING]
> 跨大版本不是原地升级:数据库 schema 与配置只在同一大版本内兼容,内置一键自更新
> 会拒绝跨大版本目标(3.0 之前的旧版没有这道门禁,不要用旧版界面的一键更新跨大版本)。
> 升级到新的大版本需要全新部署(全新 `.runtime/` 数据目录),并重新导入或重新授权
> Provider 账号与客户端 Key。
> 不支持跨大版本在线升级。升级到新的大版本时,请使用全新的 `.runtime/` 数据目录重新部署,
> 并重新导入或重新授权 Provider 账号与客户端 Key。
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
@@ -184,9 +182,10 @@ upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
release/publish <version>
```
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
仓库发版和实例升级是三个独立状态。
## 文档
+723 -209
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -25,6 +25,7 @@ tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-th
[dev-dependencies]
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
serde_json = "1"
syn = { version = "3.0.3", features = ["full"] }
tempfile = "3.27.0"
+2 -1
View File
@@ -72,7 +72,7 @@ pub async fn run() -> Result<(), BootstrapError> {
let providers = ProviderRegistry::new([openai.core_provider(), xai.core_provider()])?;
let mut core = gateway_core::initialize(store.core_ports(), providers).await?;
host.report_startup_ready("Core");
let admin = gateway_admin::initialize(
let mut admin = gateway_admin::initialize(
admin,
store.admin_ports(),
vec![openai.admin_provider(), xai.admin_provider()],
@@ -99,6 +99,7 @@ pub async fn run() -> Result<(), BootstrapError> {
plan.extend(core.take_worker_contributions());
plan.extend(openai.take_worker_contributions());
plan.extend(xai.take_worker_contributions());
plan.extend(admin.take_worker_contributions());
host.start_workers(plan, store.worker_leader_lease())?;
host.report_startup_ready("Workers");
host.serve(api.router()).await?;
+3
View File
@@ -1,6 +1,9 @@
use std::{env, error::Error, io};
use provider_openai::transport::tls::ensure_rustls_provider;
fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
ensure_rustls_provider();
let mut arguments = env::args_os().skip(1);
match arguments.next().as_deref().and_then(|value| value.to_str()) {
None | Some("serve") => {
@@ -82,6 +82,91 @@ fn gateway_admin_stays_free_of_infrastructure_dependencies() {
);
}
/// workspace 包名到冻结成员路径的映射。
const PACKAGE_TO_MEMBER: &[(&str, &str)] = &[
("codex-proxy-rs", "apps/gateway"),
("gateway-admin", "crates/gateway-admin"),
("gateway-api", "crates/gateway-api"),
("gateway-core", "crates/gateway-core"),
("gateway-host", "crates/gateway-host"),
("gateway-protocol", "crates/gateway-protocol"),
("gateway-store", "crates/gateway-store"),
("provider-openai", "crates/providers/openai"),
("provider-xai", "crates/providers/xai"),
];
/// 冻结的 workspace 内部运行时依赖边;新增/删除任何边都必须同步本表。
const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
("codex-proxy-rs", "gateway-admin"),
("codex-proxy-rs", "gateway-api"),
("codex-proxy-rs", "gateway-core"),
("codex-proxy-rs", "gateway-host"),
("codex-proxy-rs", "gateway-store"),
("codex-proxy-rs", "provider-openai"),
("codex-proxy-rs", "provider-xai"),
("gateway-admin", "gateway-core"),
("gateway-api", "gateway-admin"),
("gateway-api", "gateway-core"),
("gateway-api", "gateway-protocol"),
("gateway-host", "gateway-admin"),
("gateway-host", "gateway-core"),
("gateway-store", "gateway-admin"),
("gateway-store", "gateway-core"),
("provider-openai", "gateway-admin"),
("provider-openai", "gateway-core"),
("provider-openai", "gateway-protocol"),
("provider-xai", "gateway-admin"),
("provider-xai", "gateway-core"),
("provider-xai", "gateway-protocol"),
];
#[test]
fn workspace_internal_dependency_edges_match_frozen_dag() {
let metadata = cargo_metadata_json();
let packages = metadata["packages"]
.as_array()
.expect("cargo metadata packages");
let internal_names: std::collections::BTreeSet<&str> =
PACKAGE_TO_MEMBER.iter().map(|(name, _)| *name).collect();
let mut actual: Vec<(String, String)> = Vec::new();
for package in packages {
let name = package["name"].as_str().expect("package name");
if !internal_names.contains(name) {
continue;
}
for dependency in package["dependencies"].as_array().into_iter().flatten() {
let dep_name = dependency["name"].as_str().expect("dependency name");
if !internal_names.contains(dep_name) {
continue;
}
let is_runtime_edge = dependency["dep_kinds"]
.as_array()
.map(|kinds| {
kinds
.iter()
.any(|kind| kind["kind"].as_str() == Some("normal"))
})
.unwrap_or(true);
if is_runtime_edge {
actual.push((name.to_owned(), dep_name.to_owned()));
}
}
}
let mut expected: Vec<(String, String)> = ALLOWED_INTERNAL_EDGES
.iter()
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
.collect();
actual.sort();
expected.sort();
assert_eq!(
actual, expected,
"workspace internal dependency edges diverged from the frozen DAG"
);
}
#[test]
fn production_sources_do_not_host_tests() {
for member in WORKSPACE_MEMBERS {
@@ -107,6 +192,20 @@ fn backend_root() -> PathBuf {
.expect("backend workspace root")
}
fn cargo_metadata_json() -> serde_json::Value {
let output = std::process::Command::new("cargo")
.args(["metadata", "--format-version", "1", "--no-deps"])
.current_dir(backend_root())
.output()
.expect("run cargo metadata");
assert!(
output.status.success(),
"cargo metadata failed: {}",
String::from_utf8_lossy(&output.stderr)
);
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
}
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束。
fn dependency_names(member: &str) -> Vec<String> {
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
+2 -2
View File
@@ -135,7 +135,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
#[test]
fn config_loader_should_reject_invalid_codex_core_version() {
assert_rejected(valid_config().replace("codex_version: '0.145.0'", "codex_version: 'latest'"));
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
}
#[test]
@@ -154,7 +154,7 @@ fn config_loader_should_reject_zero_server_port() {
#[test]
fn config_loader_should_reject_invalid_desktop_profile_fields() {
assert_rejected(valid_config().replace("desktop_build: '5828'", "desktop_build: 'build'"));
assert_rejected(valid_config().replace("desktop_build: '6119'", "desktop_build: 'build'"));
}
fn assert_rejected(config: String) {
+5
View File
@@ -26,9 +26,14 @@ subtle = "2.6.1"
thiserror = "2.0.18"
tracing = "0.1.44"
uuid = { version = "1.23.3", features = ["v7"] }
cron = "0.17"
chrono-tz = "0.10"
tokio = { version = "1.52", features = ["rt", "time", "macros"] }
[dev-dependencies]
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
sha2 = "0.11"
tempfile = "3"
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
[lints]
@@ -0,0 +1,4 @@
//! 备份纯策略与 Worker 任务。
pub mod policy;
pub mod task;
@@ -0,0 +1,161 @@
//! 备份纯策略:Cron 推进与保留候选。
//!
//! 本模块不持有任何端口或对象,是 `BackupService` 与 `BackupTask` 共用的纯函数集。
use std::str::FromStr;
use chrono::{DateTime, Duration, TimeZone as _, Utc};
use chrono_tz::Tz;
use cron::Schedule;
use crate::model::backup::{BackupError, BackupRecord, BackupStatus, code};
/// 向历史回溯的最大范围(分钟)。覆盖 Feb 29 + 特定星期等罕见组合的最长间隔。
const MAX_BACKDATE_MINUTES: i64 = 40 * 366 * 24 * 60;
/// 已解析并校验的备份计划。
#[derive(Debug, Clone)]
pub struct BackupSchedule {
schedule: Schedule,
timezone: Tz,
}
impl BackupSchedule {
/// 解析并校验 5 段 Cron 与 IANA 时区。
///
/// # Errors
///
/// Cron 非法或时区不是 IANA 名称时返回 [`BackupError`]。
pub fn parse(cron_expression: &str, schedule_timezone: &str) -> Result<Self, BackupError> {
let expression = format!("0 {cron_expression}");
let schedule = Schedule::from_str(&expression).map_err(|_| {
BackupError::new(code::INVALID_CRON, "Cron 表达式必须是 5 段格式".to_owned())
})?;
let timezone = schedule_timezone.parse::<Tz>().map_err(|_| {
BackupError::new(code::INVALID_TIMEZONE, "时区必须是 IANA 名称".to_owned())
})?;
Ok(Self { schedule, timezone })
}
/// 返回最接近 `at`(含 `at` 所在分钟)的最近一次触发时间。
///
/// 若过去 `MAX_BACKDATE_MINUTES` 内没有任何触发(极罕见的稀疏计划或超长停机),
/// 返回 `None`,调用方不应为历史点补任务。
#[must_use]
pub fn last_firing_at_or_before(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
let high_min = at.timestamp().div_euclid(60);
let low_min = high_min.saturating_sub(MAX_BACKDATE_MINUTES);
// 检查窗口内是否存在触发;不存在则直接返回 None。
let low_instant = instant(low_min, self.timezone)?;
if self
.schedule
.after(&low_instant)
.next()
.is_none_or(|firing| firing.timestamp() > high_min * 60 + 59)
{
return None;
}
// 二分查找最小的 u(分钟)使 after(u) > high;prev = after(u - 1 分钟)。
let mut lo = low_min;
let mut hi = high_min;
while lo < hi {
let mid = lo + (hi - lo) / 2;
if self
.firing_after(mid)
.is_some_and(|firing| firing > high_min)
{
hi = mid;
} else {
lo = mid + 1;
}
}
self.firing_after(lo - 1)
.filter(|&firing| firing <= high_min)
.and_then(|firing| Utc.timestamp_opt(firing * 60, 0).single())
}
/// 返回第一个严格晚于 `at` 的触发时间。
#[must_use]
pub fn next_after(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
let local = at.with_timezone(&self.timezone);
self.schedule
.after(&local)
.next()
.map(|firing| firing.with_timezone(&Utc))
}
/// `after(u 分钟)` 返回的第一个触发时间(分钟索引),用于二分搜索。
fn firing_after(&self, minute_index: i64) -> Option<i64> {
let instant = instant(minute_index, self.timezone)?;
self.schedule
.after(&instant)
.next()
.map(|firing| firing.timestamp().div_euclid(60))
}
}
/// 构造指定分钟索引(UTC epoch 分钟)在目标时区下的 `DateTime`。
fn instant(minute_index: i64, timezone: Tz) -> Option<DateTime<Tz>> {
timezone.timestamp_opt(minute_index * 60, 0).single()
}
/// 一条记录被纳入删除流程的原因。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RetentionReason {
/// 超过 `retention_days`。
Days,
/// 超过 `retention_count`。
Count,
}
/// 保留决策。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetentionDecision {
pub record_id: String,
pub reason: RetentionReason,
}
/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。
///
/// 规则(`docs/s3-backup-design-audit.md` §4.4):
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理。
/// - 自动计划备份满足任一启用阈值即可进入删除。
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)。
/// - 手动备份不受自动阈值影响;无 TTL,仅由管理员手动删除。
#[must_use]
pub fn decide_retention(
retention_days: u32,
retention_count: u32,
now: DateTime<Utc>,
scheduled_completed_desc: &[BackupRecord],
) -> Vec<RetentionDecision> {
let mut decisions = Vec::new();
for (index, record) in scheduled_completed_desc.iter().enumerate() {
if record.status != BackupStatus::Completed {
continue;
}
// 最近一个成功的计划备份永远保留。
if index == 0 {
continue;
}
let expired_by_days = retention_days > 0
&& record.completed_at.is_some_and(|completed_at| {
completed_at <= now - Duration::days(i64::from(retention_days))
});
let exceeded_count =
retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX);
if expired_by_days || exceeded_count {
decisions.push(RetentionDecision {
record_id: record.id.clone(),
reason: if expired_by_days {
RetentionReason::Days
} else {
RetentionReason::Count
},
});
}
}
decisions
}
@@ -0,0 +1,602 @@
//! Backup Worker 贡献:单个可取消 `DaemonTask` 承担调度、执行、删除收敛与保留清理。
//!
//! 当前部署边界是单副本,因此不需要 Redis lease、fencing token 或 heartbeat。长时间
//! `pg_dump` 与上传由本 Daemon 自身持有;Host 只负责 panic 后重启、健康与关闭。
use std::sync::Arc;
use std::time::Duration;
use chrono::{DateTime, Utc};
use futures::future::BoxFuture;
use gateway_core::{
engine::CancellationToken,
task::{DaemonTask, WorkerTaskError},
};
use tracing::{info, warn};
use crate::{
model::backup::{
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStorageConfig,
BackupTriggerKind, build_backup_seed, code,
},
ports::backup::{
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpRequest,
StatusTransitionUpdate, UploadObjectRequest,
},
};
impl From<BackupError> for WorkerTaskError {
fn from(error: BackupError) -> Self {
WorkerTaskError::safe(error.message())
}
}
use super::policy::{BackupSchedule, decide_retention};
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待。
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
/// 每个循环最多执行的保留删除数量。
const RETENTION_BATCH_SIZE: usize = 10;
/// 保留扫描一次读取的 completed 计划备份上限。
const RETENTION_SCAN_LIMIT: u32 = 1000;
/// 每个循环最多完成的待删除记录数量。
const PENDING_DELETION_BATCH: u32 = 20;
/// 备份 Daemon 任务。
pub struct BackupTask {
repository: Arc<dyn BackupRepository>,
dump: Arc<dyn DatabaseDumpPort>,
object_store: Arc<dyn BackupObjectStorePort>,
}
impl BackupTask {
/// 组合仓储、导出器与对象存储适配器。
#[must_use]
pub fn new(
repository: Arc<dyn BackupRepository>,
dump: Arc<dyn DatabaseDumpPort>,
object_store: Arc<dyn BackupObjectStorePort>,
) -> Self {
Self {
repository,
dump,
object_store,
}
}
}
impl DaemonTask for BackupTask {
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move { self.run_daemon(cancellation).await })
}
}
impl BackupTask {
async fn run_daemon(&self, cancellation: CancellationToken) -> Result<(), WorkerTaskError> {
loop {
if cancellation.is_cancelled() {
return Ok(());
}
self.run_cycle(&cancellation).await?;
tokio::select! {
_ = cancellation.cancelled() => return Ok(()),
_ = tokio::time::sleep(BACKUP_LOOP_INTERVAL) => {}
}
}
}
/// 执行一个周期:推进计划、恢复中间态、删除收敛、领取执行一个任务、保留清理。
///
/// 对外暴露以便集成测试逐周期驱动;daemon 循环内部复用。
///
/// # Errors
///
/// 仓储或基础设施不可用、任务执行失败时返回 [`WorkerTaskError`]。
pub async fn run_cycle(&self, cancellation: &CancellationToken) -> Result<(), WorkerTaskError> {
if cancellation.is_cancelled() {
return Ok(());
}
self.advance_schedule(Utc::now()).await?;
self.recover_intermediate(cancellation).await?;
self.finish_pending_deletions().await?;
self.process_next_task(cancellation).await?;
self.run_retention_batch(Utc::now()).await?;
Ok(())
}
/// 推进到期计划并幂等插入 scheduled 任务。
async fn advance_schedule(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
if !settings.schedule_enabled {
return Ok(());
}
let cron = settings.cron_expression.as_deref().unwrap_or_default();
let timezone = settings.schedule_timezone.as_deref().unwrap_or_default();
if cron.is_empty() || timezone.is_empty() {
return Ok(());
}
let schedule = BackupSchedule::parse(cron, timezone)
.map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?;
let due = match settings.next_run_at {
Some(next_run_at) if next_run_at <= now => true,
Some(_) => false,
None => {
// 刚启用计划但尚无游标:直接初始化。
if let Some(next_run_at) = schedule.next_after(now) {
let _ = self
.repository
.advance_schedule_cursor(next_run_at, cron, timezone)
.await
.map_err(repo_error)?;
}
return Ok(());
}
};
if !due {
return Ok(());
}
let scheduled_at = match schedule.last_firing_at_or_before(now) {
Some(scheduled_at) => scheduled_at,
None => {
// 无法从计划推导最近到期时间:推进游标避免反复触发。
if let Some(next_run_at) = schedule.next_after(now) {
let _ = self
.repository
.advance_schedule_cursor(next_run_at, cron, timezone)
.await
.map_err(repo_error)?;
}
return Ok(());
}
};
let prefix = settings.prefix.as_deref().unwrap_or_default();
let expires_at = if settings.retention_days > 0 {
Some(now + chrono::Duration::days(i64::from(settings.retention_days)))
} else {
None
};
let seed = build_backup_seed(
BackupTriggerKind::Scheduled,
Some(scheduled_at),
prefix,
now,
expires_at,
)
.map_err(infra_error)?;
let inserted = self
.repository
.insert_scheduled_record(seed)
.await
.map_err(repo_error)?;
if inserted {
info!(scheduled_at = %scheduled_at, "计划备份任务已创建");
} else {
warn!(scheduled_at = %scheduled_at, "计划时间点冲突,跳过并推进游标");
}
if let Some(next_run_at) = schedule.next_after(now) {
let _ = self
.repository
.advance_schedule_cursor(next_run_at, cron, timezone)
.await
.map_err(repo_error)?;
}
Ok(())
}
/// 恢复中间状态(dumping/uploading/deleting)。
async fn recover_intermediate(
&self,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
let records = self
.repository
.list_intermediate_records()
.await
.map_err(repo_error)?;
for record in records {
match record.status {
BackupStatus::Dumping => self.recover_dumping(&record, cancellation).await?,
BackupStatus::Uploading => self.recover_uploading(&record, cancellation).await?,
_ => {}
}
}
Ok(())
}
/// dumping:有完整暂存则继续上传,否则清理暂存并标记失败。
async fn recover_dumping(
&self,
record: &BackupRecord,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
match self.dump.inspect_staging(&record.id).await? {
Some(artifact) => {
let now = Utc::now();
let update = StatusTransitionUpdate {
size_bytes: Some(artifact.size_bytes),
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
BackupStatus::Uploading,
update,
now,
)
.await
.map_err(repo_error)?;
self.upload_and_verify(
record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
now,
cancellation,
)
.await?;
}
None => {
let _ = self.dump.cleanup_staging(&record.id).await;
self.fail_task(record, code::PG_DUMP_FAILED, "导出进程中断,暂存归档不完整")
.await?;
}
}
Ok(())
}
/// uploading:远端匹配则补记完成,有完整暂存则重试上传,否则失败。
async fn recover_uploading(
&self,
record: &BackupRecord,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Ok(());
};
if let Some(remote) = self
.object_store
.head_object(&config, &record.object_key)
.await?
&& record.size_bytes == Some(remote.size_bytes)
&& record.sha256.as_deref() == Some(remote.sha256.as_str())
{
let now = Utc::now();
self.repository
.transition_status(
&record.id,
BackupStatus::Uploading,
BackupStatus::Completed,
StatusTransitionUpdate::default(),
now,
)
.await
.map_err(repo_error)?;
return Ok(());
}
match self.dump.inspect_staging(&record.id).await? {
Some(artifact) => {
self.upload_and_verify(
record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
Utc::now(),
cancellation,
)
.await
}
None => {
self.fail_task(
record,
code::S3_UPLOAD_FAILED,
"远端对象缺失且本地无完整暂存",
)
.await
}
}
}
/// 完成待删除记录(deleting → DeleteObject → 硬删除)。
async fn finish_pending_deletions(&self) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Ok(());
};
let records = self
.repository
.list_pending_deletions(PENDING_DELETION_BATCH)
.await
.map_err(repo_error)?;
for record in records {
if self
.object_store
.delete_object(&config, &record.object_key)
.await
.is_ok()
{
self.repository
.delete_record(&record.id)
.await
.map_err(repo_error)?;
}
}
Ok(())
}
/// 领取一个 queued 任务并执行导出、上传与远端校验。
async fn process_next_task(
&self,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
let now = Utc::now();
let Some(record) = self
.repository
.claim_next_queued(now)
.await
.map_err(repo_error)?
else {
return Ok(());
};
info!(
backup_id = %record.id,
attempt = record.attempt_count,
"领取备份任务"
);
if let Err(error) = self.execute_task(&record, cancellation).await {
warn!(backup_id = %record.id, error = %error, "备份任务执行失败");
}
Ok(())
}
/// 执行单个任务;内部处理全部状态迁移。
async fn execute_task(
&self,
record: &BackupRecord,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
if BackupStorageConfig::from_settings(&settings).is_none() {
return self
.fail_task(record, code::INVALID_CONFIG, "备份存储配置不完整")
.await;
}
if !settings.storage_verified() {
return self
.fail_task(record, code::INVALID_CONFIG, "备份存储配置尚未通过连接测试")
.await;
}
let now = Utc::now();
let artifact = match self
.dump
.dump(DumpRequest {
backup_id: record.id.clone(),
cancellation: cancellation.clone(),
})
.await
{
Ok(artifact) => artifact,
Err(error) => {
let (error_code, message) = classify_dump_error(error);
return self.fail_task(record, error_code, message).await;
}
};
// dumping → uploading,持久化 size/sha256。
let update = StatusTransitionUpdate {
size_bytes: Some(artifact.size_bytes),
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
BackupStatus::Uploading,
update,
now,
)
.await
.map_err(repo_error)?;
self.upload_and_verify(
record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
now,
cancellation,
)
.await?;
let _ = self.dump.cleanup_staging(&record.id).await;
Ok(())
}
/// 上传 + 远端校验,内部完成终态迁移。
async fn upload_and_verify(
&self,
record: &BackupRecord,
source: &std::path::Path,
size_bytes: u64,
sha256: &str,
now: DateTime<Utc>,
cancellation: &CancellationToken,
) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Ok(());
};
let store = self.object_store.clone();
let metadata =
BackupObjectMetadata::new(record.id.clone(), sha256.to_owned(), now, size_bytes);
if let Err(error) = store
.upload_file(
&config,
UploadObjectRequest {
object_key: record.object_key.clone(),
source: source.to_path_buf(),
metadata,
cancellation: cancellation.clone(),
},
)
.await
{
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "上传到对象存储失败")
.await?;
return Ok(());
}
match store.head_object(&config, &record.object_key).await {
Ok(Some(remote)) if remote.size_bytes == size_bytes && remote.sha256 == sha256 => {
if self
.repository
.transition_status(
&record.id,
BackupStatus::Uploading,
BackupStatus::Completed,
StatusTransitionUpdate::default(),
Utc::now(),
)
.await
.map_err(repo_error)?
.is_some()
{
info!(
backup_id = %record.id,
size_bytes,
"备份完成"
);
}
}
Ok(_) => {
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(
record,
code::REMOTE_VERIFICATION_FAILED,
"远端对象大小或校验值不一致",
)
.await?;
}
Err(error) => {
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "远端对象校验失败")
.await?;
}
}
Ok(())
}
/// 把活跃任务写入失败终态并清理暂存。
async fn fail_task(
&self,
record: &BackupRecord,
error_code: &'static str,
message: impl Into<String>,
) -> Result<(), WorkerTaskError> {
let now = Utc::now();
let update = StatusTransitionUpdate {
error_code: Some(error_code.to_string()),
error_message: Some(message.into()),
completed_at: Some(now),
..Default::default()
};
let _ = self
.repository
.transition_status(&record.id, record.status, BackupStatus::Failed, update, now)
.await
.map_err(repo_error)?;
let _ = self.dump.cleanup_staging(&record.id).await;
Ok(())
}
/// 执行一小批到期保留清理:先处理 `expires_at` 已到期的记录,再按天数/份数清理计划备份。
async fn run_retention_batch(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
// 1. expires_at 已到期的记录(手动或计划)无条件进入删除。
let expired = self
.repository
.list_expired_records(RETENTION_BATCH_SIZE as u32)
.await
.map_err(repo_error)?;
for record in expired {
let Some(record) = self
.repository
.transition_to_deleting(&record.id, now)
.await
.map_err(repo_error)?
else {
continue;
};
self.finalize_deletion(&record).await?;
}
// 2. 计划备份按 retentionDays / retentionCount 清理。
let records = self
.repository
.list_scheduled_completed_desc(RETENTION_SCAN_LIMIT)
.await
.map_err(repo_error)?;
let decisions = decide_retention(
settings.retention_days,
settings.retention_count,
now,
&records,
);
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
let Some(record) = self
.repository
.transition_to_deleting(&decision.record_id, now)
.await
.map_err(repo_error)?
else {
continue;
};
self.finalize_deletion(&record).await?;
}
Ok(())
}
/// 对已进入 deleting 的记录执行对象删除并硬删除记录行。
async fn finalize_deletion(&self, record: &BackupRecord) -> Result<(), WorkerTaskError> {
let settings = self.repository.load_settings().await.map_err(repo_error)?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Ok(());
};
if self
.object_store
.delete_object(&config, &record.object_key)
.await
.is_ok()
{
self.repository
.delete_record(&record.id)
.await
.map_err(repo_error)?;
}
Ok(())
}
}
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
match error.code() {
code::CANCELLED => (code::CANCELLED, "备份已取消"),
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
}
}
fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError {
WorkerTaskError::safe(format!("backup store failed: {error}"))
}
fn infra_error(error: BackupError) -> WorkerTaskError {
WorkerTaskError::safe(error.message())
}
+61 -6
View File
@@ -2,23 +2,28 @@
//!
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现。
use std::{fmt, path::Path, sync::Arc};
use std::{fmt, path::Path, sync::Arc, time::Duration};
use gateway_core::{
engine::probe::AccountProbe,
routing::{ProviderKind, snapshot::SnapshotControl},
task::{
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
WorkerRunnable,
},
};
use secrecy::{ExposeSecret as _, SecretString};
use serde::Deserialize;
pub mod backup;
pub mod model;
pub mod ports;
mod use_case;
pub use use_case::{
accounts::AccountsService, auth::AuthService, client_keys::ClientKeyService,
observability::ObservabilityService, openai::OpenAiService, settings::SettingsService,
system::SystemService, xai::XaiService,
accounts::AccountsService, auth::AuthService, backup::BackupService,
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
settings::SettingsService, system::SystemService, xai::XaiService,
};
use model::{AdminError, AdminErrorKind};
@@ -28,7 +33,7 @@ use ports::{
system::SystemOperations,
};
use use_case::{
accounts::DefaultAccountsService, auth::DefaultAuthService,
accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService,
client_keys::DefaultClientKeyService, observability::DefaultObservabilityService,
openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService,
xai::DefaultXaiService,
@@ -48,6 +53,8 @@ const WEAK_INITIAL_PASSWORDS: &[&str] = &[
"codex-proxy-rs",
];
const BACKUP_WORKER_OWNER: &str = "backup";
/// 只用于首次幂等创建默认管理员的启动密码。
#[derive(Clone, Deserialize)]
#[serde(transparent)]
@@ -146,6 +153,7 @@ pub struct AdminServices {
system: Arc<dyn SystemService>,
openai: Arc<dyn OpenAiService>,
xai: Arc<dyn XaiService>,
backups: Arc<dyn BackupService>,
}
impl AdminServices {
@@ -188,11 +196,17 @@ impl AdminServices {
pub fn xai(&self) -> &dyn XaiService {
self.xai.as_ref()
}
#[must_use]
pub fn backups(&self) -> &dyn BackupService {
self.backups.as_ref()
}
}
/// Admin 初始化完成后的封闭能力包。
pub struct AdminBundle {
services: AdminServices,
worker_contributions: Vec<WorkerContribution>,
}
impl AdminBundle {
@@ -200,6 +214,11 @@ impl AdminBundle {
pub fn services(&self) -> AdminServices {
self.services.clone()
}
/// 取出 Backup Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
std::mem::take(&mut self.worker_contributions)
}
}
/// 校验配置、建立动态 Provider 注册表并完成默认管理员幂等初始化。
@@ -241,6 +260,18 @@ pub async fn initialize(
snapshot.clone(),
probe.clone(),
));
let backup_ports = store.backup();
let backups = Arc::new(DefaultBackupService::new(
backup_ports.repository(),
backup_ports.object_store(),
store.auth(),
snapshot.clone(),
));
let backup_task = backup::task::BackupTask::new(
backup_ports.repository(),
backup_ports.dump(),
backup_ports.object_store(),
);
let services = AdminServices {
auth,
accounts,
@@ -268,8 +299,32 @@ pub async fn initialize(
store.accounts(),
snapshot.clone(),
)),
backups,
};
Ok(AdminBundle { services })
let worker_contributions = backup_worker_contribution(backup_task)?;
Ok(AdminBundle {
services,
worker_contributions,
})
}
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`。
fn backup_worker_contribution(
task: backup::task::BackupTask,
) -> Result<Vec<WorkerContribution>, AdminError> {
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
.map_err(|_| AdminError::internal("Backup worker id is invalid"))?;
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
.map_err(|_| AdminError::internal("Backup worker restart policy is invalid"))?;
let registration = WorkerRegistration::try_new(
id,
WorkerRunnable::Daemon {
restart,
task: Box::new(task),
},
)
.map_err(|_| AdminError::internal("Backup worker registration is invalid"))?;
Ok(vec![WorkerContribution::Registration(registration)])
}
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
@@ -16,8 +16,10 @@ pub use gateway_core::engine::credential::AccountAvailability;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountStatus {
Active,
Expired,
RateLimited,
QuotaExhausted,
Expired,
Invalid,
Disabled,
Banned,
}
@@ -75,8 +77,6 @@ pub struct AccountRecord {
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_reason: Option<String>,
pub cooldown_until: Option<DateTime<Utc>>,
pub availability_observed_at: DateTime<Utc>,
pub quota_observed_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
@@ -169,6 +169,8 @@ pub struct AccountPage {
}
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountSummary {
pub total: u64,
@@ -0,0 +1,566 @@
//! S3 备份领域模型、状态机与稳定错误。
//!
//! 领域类型只携带事实,不承载基础设施细节。Secret 使用 [`secrecy::SecretString`],
//! 全流程不实现 `Serialize`,避免被错误地写入 wire 或日志。管理员身份等审计事实
//! 只存在于 `admin_audit_events`,本模型不重复保存。
use std::fmt;
use chrono::{DateTime, Utc};
use secrecy::SecretString;
use uuid::Uuid;
use super::PageSize;
/// 备份触发来源。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackupTriggerKind {
Manual,
Scheduled,
}
impl BackupTriggerKind {
/// 稳定持久化值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Manual => "manual",
Self::Scheduled => "scheduled",
}
}
/// 解析持久化值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"manual" => Some(Self::Manual),
"scheduled" => Some(Self::Scheduled),
_ => None,
}
}
}
impl fmt::Display for BackupTriggerKind {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
/// 备份任务状态机(六状态)。
///
/// 迁移 `backup_records_lifecycle_ck` 与字段空值语义共同定义合法持久化行;
/// [`Self::allows_transition_to`] 只约束领域内的显式状态迁移,两层都必须满足。
/// 删除成功后记录行被硬删除,因此不存在 `deleted` 终态。
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum BackupStatus {
Queued,
Dumping,
Uploading,
Completed,
Failed,
Deleting,
}
impl BackupStatus {
/// 稳定持久化值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Dumping => "dumping",
Self::Uploading => "uploading",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Deleting => "deleting",
}
}
/// 解析持久化值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"queued" => Some(Self::Queued),
"dumping" => Some(Self::Dumping),
"uploading" => Some(Self::Uploading),
"completed" => Some(Self::Completed),
"failed" => Some(Self::Failed),
"deleting" => Some(Self::Deleting),
_ => None,
}
}
/// 该状态是否消耗全局活跃名额。
#[must_use]
pub const fn is_active(self) -> bool {
matches!(self, Self::Queued | Self::Dumping | Self::Uploading)
}
/// 该状态是否允许进入删除流程。
#[must_use]
pub const fn can_be_deleted(self) -> bool {
matches!(self, Self::Completed | Self::Failed)
}
/// 领域内允许的显式迁移。
#[must_use]
pub fn allows_transition_to(self, target: BackupStatus) -> bool {
matches!(
(self, target),
(Self::Queued, Self::Dumping)
| (Self::Queued, Self::Failed)
| (Self::Dumping, Self::Uploading)
| (Self::Dumping, Self::Failed)
| (Self::Uploading, Self::Completed)
| (Self::Uploading, Self::Failed)
| (Self::Completed, Self::Deleting)
| (Self::Failed, Self::Deleting)
)
}
}
impl fmt::Display for BackupStatus {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
/// S3 存储配置更新命令;`secret_access_key` 为 `None` 表示保留旧值。
#[derive(Clone)]
pub struct UpdateBackupStorageCommand {
pub endpoint: String,
pub region: String,
pub bucket: String,
pub access_key_id: String,
pub secret_access_key: Option<SecretString>,
pub prefix: String,
pub force_path_style: bool,
}
impl fmt::Debug for UpdateBackupStorageCommand {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("UpdateBackupStorageCommand")
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.field("bucket", &self.bucket)
.field("access_key_id", &self.access_key_id)
.field("secret_access_key", &self.secret_access_key.is_some())
.field("prefix", &self.prefix)
.field("force_path_style", &self.force_path_style)
.finish()
}
}
/// 调度配置更新命令。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpdateBackupScheduleCommand {
pub schedule_enabled: bool,
pub cron_expression: String,
pub schedule_timezone: String,
pub retention_days: u32,
pub retention_count: u32,
}
/// 完整保存的备份设置(含 Secret)。
///
/// 只从仓储读取;不实现 `Serialize`,防止 Secret 进入 wire 或日志。
#[derive(Clone)]
pub struct BackupSettings {
pub storage_revision: u64,
pub endpoint: Option<String>,
pub region: Option<String>,
pub bucket: Option<String>,
pub access_key_id: Option<String>,
pub secret_access_key: Option<SecretString>,
pub prefix: Option<String>,
pub force_path_style: bool,
pub schedule_enabled: bool,
pub cron_expression: Option<String>,
pub schedule_timezone: Option<String>,
pub retention_days: u32,
pub retention_count: u32,
pub next_run_at: Option<DateTime<Utc>>,
pub last_verified_at: Option<DateTime<Utc>>,
pub updated_at: DateTime<Utc>,
}
impl fmt::Debug for BackupSettings {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("BackupSettings")
.field("storage_revision", &self.storage_revision)
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.field("bucket", &self.bucket)
.field("access_key_id", &self.access_key_id)
.field("secret_access_key", &self.secret_access_key.is_some())
.field("prefix", &self.prefix)
.field("force_path_style", &self.force_path_style)
.field("schedule_enabled", &self.schedule_enabled)
.field("cron_expression", &self.cron_expression)
.field("schedule_timezone", &self.schedule_timezone)
.field("retention_days", &self.retention_days)
.field("retention_count", &self.retention_count)
.field("next_run_at", &self.next_run_at)
.field("last_verified_at", &self.last_verified_at)
.field("updated_at", &self.updated_at)
.finish()
}
}
impl BackupSettings {
/// S3 存储配置是否完整(bucket/region/密钥/前缀齐全)。
#[must_use]
pub fn storage_configured(&self) -> bool {
self.endpoint.is_some()
&& self.region.is_some()
&& self.bucket.is_some()
&& self.access_key_id.is_some()
&& self.secret_access_key.is_some()
&& self.prefix.is_some()
}
/// 探测是否已对当前 revision 成功;`last_verified_at` 非空即表示当前配置已通过探针。
#[must_use]
pub fn storage_verified(&self) -> bool {
self.last_verified_at.is_some()
}
}
/// 已经校验并可用于对象存储操作的存储配置快照。
#[derive(Clone)]
pub struct BackupStorageConfig {
pub storage_revision: u64,
pub endpoint: String,
pub region: String,
pub bucket: String,
pub access_key_id: String,
pub secret_access_key: SecretString,
pub prefix: String,
pub force_path_style: bool,
}
impl fmt::Debug for BackupStorageConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("BackupStorageConfig")
.field("storage_revision", &self.storage_revision)
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.field("bucket", &self.bucket)
.field("access_key_id", &self.access_key_id)
.field("secret_access_key", &"[REDACTED]")
.field("prefix", &self.prefix)
.field("force_path_style", &self.force_path_style)
.finish()
}
}
impl BackupStorageConfig {
/// 从完整设置推导对象存储配置;设置不完整时返回 `None`。
#[must_use]
pub fn from_settings(settings: &BackupSettings) -> Option<Self> {
Some(Self {
storage_revision: settings.storage_revision,
endpoint: settings.endpoint.clone()?,
region: settings.region.clone()?,
bucket: settings.bucket.clone()?,
access_key_id: settings.access_key_id.clone()?,
secret_access_key: settings.secret_access_key.clone()?,
prefix: settings.prefix.clone()?,
force_path_style: settings.force_path_style,
})
}
}
/// 一条备份记录。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackupRecord {
pub id: String,
pub trigger_kind: BackupTriggerKind,
pub status: BackupStatus,
pub scheduled_at: Option<DateTime<Utc>>,
pub object_key: String,
pub size_bytes: Option<u64>,
pub sha256: Option<String>,
pub attempt_count: u32,
pub error_code: Option<String>,
pub error_message: Option<String>,
pub started_at: Option<DateTime<Utc>>,
pub completed_at: Option<DateTime<Utc>>,
/// 手动过期时间;创建时确定,到期自动清理。
pub expires_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
/// 备份记录分页查询。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackupRecordListQuery {
pub page: u32,
pub page_size: PageSize,
pub status: Option<BackupStatus>,
pub trigger: Option<BackupTriggerKind>,
}
/// 备份记录分页结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackupRecordPage {
pub items: Vec<BackupRecord>,
pub total: u64,
pub page: u32,
pub page_size: PageSize,
}
impl BackupRecordPage {
/// 计算总页数;空集合视为一页。
#[must_use]
pub fn total_pages(&self) -> u32 {
let size = u64::from(self.page_size.get());
if self.total == 0 {
1
} else {
self.total.div_ceil(size).min(u64::from(u32::MAX)) as u32
}
}
}
/// 创建备份任务的持久化数据;id 与对象 key 由领域生成。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackupRecordSeed {
pub id: String,
pub trigger_kind: BackupTriggerKind,
pub scheduled_at: Option<DateTime<Utc>>,
pub object_key: String,
pub expires_at: Option<DateTime<Utc>>,
}
/// 连接测试结果;只携带稳定阶段与脱敏消息。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConnectionTestResult {
pub ok: bool,
pub stage: &'static str,
pub code: Option<&'static str>,
pub message: String,
}
/// 对象存储探测阶段。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectionTestStage {
PutObject,
HeadObject,
GetObject,
DeleteObject,
}
impl ConnectionTestStage {
/// 稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::PutObject => "putObject",
Self::HeadObject => "headObject",
Self::GetObject => "getObject",
Self::DeleteObject => "deleteObject",
}
}
}
/// 上传到对象存储并期望远端校验的归档 metadata。
///
/// `size_bytes` 在上传时不写入 metadata,但 `HeadObject` 会返回远端大小,
/// 用于与本地归档校验一致。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackupObjectMetadata {
pub backup_id: String,
pub sha256: String,
pub created_at: DateTime<Utc>,
pub size_bytes: u64,
}
impl BackupObjectMetadata {
/// 构造上传 metadata。
#[must_use]
pub const fn new(
backup_id: String,
sha256: String,
created_at: DateTime<Utc>,
size_bytes: u64,
) -> Self {
Self {
backup_id,
sha256,
created_at,
size_bytes,
}
}
}
/// 短时下载地址结果。
#[derive(Clone, PartialEq, Eq)]
pub struct DownloadUrlResult {
pub url: String,
pub file_name: String,
pub expires_in: std::time::Duration,
}
impl fmt::Debug for DownloadUrlResult {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DownloadUrlResult")
.field("url", &"[REDACTED]")
.field("file_name", &self.file_name)
.field("expires_in", &self.expires_in)
.finish()
}
}
/// 备份基础设施错误:只携带稳定错误码与脱敏消息。
///
/// 该类型不构成第二套 kind 枚举;`gateway-admin` 边界统一映射为既有
/// `AdminErrorKind`。稳定 `error_code` 同时用于任务记录、日志与 UI。
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error("{message}")]
pub struct BackupError {
code: &'static str,
message: String,
}
impl BackupError {
/// 构造基础设施错误。
#[must_use]
pub const fn new(code: &'static str, message: String) -> Self {
Self { code, message }
}
/// 稳定错误码,例如 `backup.s3_auth_failed`。
#[must_use]
pub const fn code(&self) -> &'static str {
self.code
}
/// 脱敏后的可操作消息。
#[must_use]
pub fn message(&self) -> &str {
&self.message
}
}
/// 构造稳定错误码常量。
pub mod code {
/// 配置或参数非法。
pub const INVALID_CONFIG: &str = "backup.invalid_config";
/// Cron 表达式非法。
pub const INVALID_CRON: &str = "backup.invalid_cron";
/// IANA 时区非法。
pub const INVALID_TIMEZONE: &str = "backup.invalid_timezone";
/// 记录不存在。
pub const RECORD_NOT_FOUND: &str = "backup.record_not_found";
/// 已存在活跃任务,拒绝创建。
pub const ACTIVE_TASK_CONFLICT: &str = "backup.active_task_conflict";
/// 记录状态与期望不符,无法迁移。
pub const STATE_CONFLICT: &str = "backup.state_conflict";
/// 已有记录时禁止切换存储目标。
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
/// `pg_dump` 非零退出或失败。
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
/// 暂存磁盘空间不足。
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
/// S3 认证失败。
pub const S3_AUTH_FAILED: &str = "backup.s3_auth_failed";
/// S3 权限拒绝。
pub const S3_PERMISSION_DENIED: &str = "backup.s3_permission_denied";
/// S3 上传失败。
pub const S3_UPLOAD_FAILED: &str = "backup.s3_upload_failed";
/// 远端对象校验失败。
pub const REMOTE_VERIFICATION_FAILED: &str = "backup.remote_verification_failed";
/// 任务被取消。
pub const CANCELLED: &str = "backup.cancelled";
/// 仓储或对象存储暂不可用。
pub const STORE_UNAVAILABLE: &str = "backup.store_unavailable";
/// 上游 S3 服务返回无效或失败响应。
pub const UPSTREAM_FAILURE: &str = "backup.upstream_failure";
}
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`。
///
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
/// 输出统一使用 `/` 分隔且不含空段。
pub fn build_object_key(
prefix: &str,
backup_id: &str,
at: DateTime<Utc>,
) -> Result<String, BackupError> {
let prefix = normalize_prefix(prefix)?;
Ok(format!(
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
at.format("%Y"),
at.format("%m"),
at.format("%d"),
at.format("%Y%m%d_%H%M%S"),
short_backup_id(backup_id),
))
}
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致。
#[must_use]
pub fn build_download_file_name(backup_id: &str) -> String {
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
}
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex。
#[must_use]
fn short_backup_id(backup_id: &str) -> String {
let hex = backup_id.strip_prefix("backup_").unwrap_or(backup_id);
hex.get(..8).unwrap_or(hex).to_owned()
}
/// 规范化并校验对象 prefix。
///
/// 返回去掉尾部 `/` 的规范 prefix;空串、前导 `/`、`..` 段、反斜杠和控制字符均拒绝。
pub fn normalize_prefix(prefix: &str) -> Result<String, BackupError> {
if prefix.is_empty() || prefix.starts_with('/') {
return Err(BackupError::new(
code::INVALID_CONFIG,
"存储对象前缀必须非空且不能以前导 / 开头".to_owned(),
));
}
if prefix.bytes().any(|byte| byte.is_ascii_control()) || prefix.contains('\\') {
return Err(BackupError::new(
code::INVALID_CONFIG,
"存储对象前缀包含非法字符".to_owned(),
));
}
if prefix.split('/').any(|segment| segment == "..") {
return Err(BackupError::new(
code::INVALID_CONFIG,
"存储对象前缀不能包含 .. 路径段".to_owned(),
));
}
Ok(prefix.trim_end_matches('/').to_owned())
}
/// 生成新的备份记录 id。
#[must_use]
pub fn new_backup_id() -> String {
format!("backup_{}", Uuid::now_v7().simple())
}
/// 从触发类型与计划时间构造可持久化 seed;id 与对象 key 一次生成保持一致。
pub fn build_backup_seed(
trigger_kind: BackupTriggerKind,
scheduled_at: Option<DateTime<Utc>>,
prefix: &str,
at: DateTime<Utc>,
expires_at: Option<DateTime<Utc>>,
) -> Result<BackupRecordSeed, BackupError> {
let id = new_backup_id();
let object_key = build_object_key(prefix, &id, at)?;
Ok(BackupRecordSeed {
id,
trigger_kind,
scheduled_at,
object_key,
expires_at,
})
}
@@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroU64};
pub mod accounts;
pub mod auth;
pub mod backup;
pub mod client_keys;
pub mod observability;
pub mod provider_credentials;
@@ -50,6 +51,11 @@ impl AdminError {
Self::new(AdminErrorKind::Invalid, message)
}
#[must_use]
pub fn bad_request(message: impl Into<String>) -> Self {
Self::new(AdminErrorKind::Invalid, message)
}
#[must_use]
pub fn not_found(message: impl Into<String>) -> Self {
Self::new(AdminErrorKind::NotFound, message)
@@ -60,6 +66,11 @@ impl AdminError {
Self::new(AdminErrorKind::Conflict, message)
}
#[must_use]
pub fn bad_gateway(message: impl Into<String>) -> Self {
Self::new(AdminErrorKind::BadGateway, message)
}
#[must_use]
pub fn unavailable(message: impl Into<String>) -> Self {
Self::new(AdminErrorKind::Unavailable, message)
@@ -204,42 +204,22 @@ impl DecimalAmount {
/// 精确相加两个 `numeric(20,10)` 金额。
#[must_use]
pub fn checked_add(&self, other: &Self) -> Option<Self> {
let sum = scaled_amount(&self.0)?.checked_add(scaled_amount(&other.0)?)?;
Self::from_scaled(sum)
Self::from_decimal(self.to_decimal()?.checked_add(other.to_decimal()?)?)
}
/// 将金额按非零请求数均分,保留最多十位小数。
#[must_use]
pub fn checked_div_u64(&self, divisor: u64) -> Option<Self> {
let divisor = u128::from(divisor);
(divisor != 0)
.then(|| scaled_amount(&self.0)?.checked_div(divisor))
.flatten()
.and_then(Self::from_scaled)
Self::from_decimal(self.to_decimal()?.checked_div_u64(divisor)?)
}
fn from_scaled(value: u128) -> Option<Self> {
if value >= 10_u128.pow(20) {
return None;
}
let whole = value / 10_u128.pow(10);
let fraction = value % 10_u128.pow(10);
let value = if fraction == 0 {
whole.to_string()
} else {
format!("{whole}.{fraction:010}")
.trim_end_matches('0')
.to_owned()
};
Some(Self(value))
fn to_decimal(&self) -> Option<gateway_core::accounting::Decimal> {
self.0.parse().ok()
}
}
fn scaled_amount(value: &str) -> Option<u128> {
let (whole, fraction) = value.split_once('.').unwrap_or((value, ""));
let whole = whole.parse::<u128>().ok()?;
let fraction = format!("{fraction:0<10}").parse::<u128>().ok()?;
whole.checked_mul(10_u128.pow(10))?.checked_add(fraction)
fn from_decimal(value: gateway_core::accounting::Decimal) -> Option<Self> {
Some(Self(value.canonical()))
}
}
impl FromStr for DecimalAmount {
@@ -460,13 +440,20 @@ pub struct RequestMetricPoint {
}
/// 账号池容量统计。
///
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
///
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
/// 以便状态详情展示和诊断。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountPoolMetrics {
pub total: u64,
pub enabled: u64,
pub unavailable: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub disabled: u64,
@@ -645,6 +632,9 @@ pub struct UsageAttempt {
pub operation: String,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
pub provider_account_email: Option<String>,
pub provider_account_authentication_kind: Option<String>,
pub upstream_model_id: Option<String>,
pub upstream_transport: Option<String>,
pub upstream_send_state: Option<String>,
@@ -705,6 +695,7 @@ pub struct UsageSummary {
/// 单个诊断维度值的聚合结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DiagnosticObservation {
pub key: String,
pub name: String,
pub request_count: u64,
pub success_count: u64,
@@ -712,6 +703,7 @@ pub struct DiagnosticObservation {
pub attempt_count: u64,
pub total_tokens: u64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCost>,
}
@@ -726,8 +718,12 @@ pub struct OpsError {
pub client_api_key_ref: Option<String>,
pub component: String,
pub operation: String,
pub endpoint: Option<String>,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
pub provider_account_email: Option<String>,
pub provider_account_authentication_kind: Option<String>,
pub upstream_model_id: Option<String>,
pub upstream_transport: Option<String>,
pub failure_kind: String,
@@ -1021,6 +1017,7 @@ pub struct UsageInsights {
/// 已完成分母计算的诊断项。
#[derive(Debug, Clone, PartialEq)]
pub struct DiagnosticsItem {
pub key: String,
pub name: String,
pub request_count: u64,
pub success_count: u64,
@@ -1028,6 +1025,7 @@ pub struct DiagnosticsItem {
pub error_rate: f64,
pub request_share: f64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub estimated_cost: Option<DecimalAmount>,
pub attempt_count: u64,
pub total_tokens: u64,
@@ -153,8 +153,6 @@ pub struct PreparedCredentialCreate {
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_reason: Option<String>,
pub cooldown_until: Option<DateTime<Utc>>,
pub availability_observed_at: DateTime<Utc>,
}
@@ -573,6 +571,7 @@ pub struct ProviderQuotaWindow {
pub window_seconds: Option<u64>,
pub used_percent: Option<f64>,
pub reset_at: Option<DateTime<Utc>>,
pub limit_reached: bool,
pub local_usage: Option<AccountUsage>,
pub provider_data: Option<ProviderDocument>,
}
@@ -591,6 +590,8 @@ pub struct ProviderQuota {
pub observed_at: Option<DateTime<Utc>>,
pub refresh_token_expires_at: Option<DateTime<Utc>>,
pub windows: Vec<ProviderQuotaWindow>,
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
pub limit_reached: bool,
pub provider_data: Option<ProviderDocument>,
}
@@ -11,14 +11,8 @@ use super::Revision;
/// 客户端模型到上游模型的全局精确映射。
pub type ModelMappings = BTreeMap<PublicModelId, UpstreamModelId>;
/// 账号调度策略。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RotationStrategy {
Smart,
QuotaResetPriority,
RoundRobin,
Sticky,
}
/// 账号调度策略;由 Core 拥有稳定值与 wire 映射。
pub use gateway_core::engine::credential::RotationStrategy;
/// 完整运行设置事实。
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -0,0 +1,442 @@
//! 备份控制面所需的持久化与基础设施能力。
//!
//! 端口方法只接受/返回领域类型与既有错误体系:`BackupRepository` 复用
//! `AdminStoreError`;`DatabaseDumpPort` 与 `BackupObjectStorePort` 返回只携带稳定
//! 错误码的 [`BackupError`]。`gateway-store` 实现全部端口;`gateway-admin` 拥有
//! 状态机、Cron、保留策略与 BackupTask 算法。
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use crate::model::Revision;
use crate::model::{
MutationContext,
backup::{
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, ConnectionTestResult,
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
},
};
use crate::ports::store::AdminStoreResult;
/// Worker 状态迁移携带的字段更新;`None` 字段不写入。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct StatusTransitionUpdate {
pub size_bytes: Option<u64>,
pub sha256: Option<String>,
pub error_code: Option<String>,
pub error_message: Option<String>,
pub completed_at: Option<DateTime<Utc>>,
}
/// 备份配置、任务与计划游标的 PostgreSQL 仓储端口。
#[async_trait]
pub trait BackupRepository: Send + Sync {
/// 读取完整设置(含 Secret)。
async fn load_settings(&self) -> AdminStoreResult<BackupSettings>;
/// 同事务更新 S3 存储配置、递增 revision、清空验证状态并写审计事件。
///
/// 已有记录时禁止切换 endpoint/region/bucket/path-style(存储身份锁定)。
/// 返回更新后的设置与新的全局 `config_revision`。
async fn update_storage_settings(
&self,
command: UpdateBackupStorageCommand,
context: &MutationContext,
) -> AdminStoreResult<(BackupSettings, Revision)>;
/// 同事务更新调度配置、推进计划游标并写审计事件。
async fn update_schedule_settings(
&self,
command: UpdateBackupScheduleCommand,
next_run_at: Option<DateTime<Utc>>,
context: &MutationContext,
) -> AdminStoreResult<BackupSettings>;
/// CAS 记录探针成功;`storage_revision` 与当前配置不一致时丢弃结果(返回 `false`)。
async fn record_verification(
&self,
storage_revision: u64,
at: DateTime<Utc>,
) -> AdminStoreResult<bool>;
/// 插入 queued 任务;存在活跃任务时返回 `backup.active_task_conflict`。
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord>;
/// 幂等插入 scheduled 任务;同一时间点或活跃冲突时返回 `false`(调用方推进游标即可)。
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool>;
/// 分页查询记录。
async fn list_backup_records(
&self,
query: BackupRecordListQuery,
) -> AdminStoreResult<BackupRecordPage>;
/// 读取单条记录。
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>>;
/// 读取中间状态记录(dumping/uploading),供启动恢复。
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>>;
/// 读取等待完成墓碑的记录(status = deleting)。
async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
/// 读取 `expires_at` 已到期且非活跃的记录(completed/failed),供保留清理。
async fn list_expired_records(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
/// 原子领取最旧的 queued 任务并迁移到 dumping;无任务返回 `None`。
async fn claim_next_queued(&self, now: DateTime<Utc>)
-> AdminStoreResult<Option<BackupRecord>>;
/// 条件迁移状态:要求当前状态一致;不匹配返回 `None`。
async fn transition_status(
&self,
id: &str,
from: BackupStatus,
to: BackupStatus,
update: StatusTransitionUpdate,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>>;
/// 迁移到删除流程(completed/failed → deleting);供管理员与保留策略调用。
async fn transition_to_deleting(
&self,
id: &str,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>>;
/// 远端删除成功后硬删除记录行。
async fn delete_record(&self, id: &str) -> AdminStoreResult<()>;
/// 条件推进计划游标;仅当计划表达式和时区仍是 `expected_*` 时才写入,返回是否推进。
async fn advance_schedule_cursor(
&self,
next_run_at: DateTime<Utc>,
expected_cron: &str,
expected_timezone: &str,
) -> AdminStoreResult<bool>;
/// 按完成时间倒序读取 completed 计划备份,供保留策略扫描(最多 `limit` 条)。
async fn list_scheduled_completed_desc(
&self,
limit: u32,
) -> AdminStoreResult<Vec<BackupRecord>>;
}
/// `pg_dump` 导出与本地暂存端口。
#[async_trait]
pub trait DatabaseDumpPort: Send + Sync {
/// 创建归档;完成后原子落盘。取消时终止子进程并清理部分文件。
async fn dump(&self, request: DumpRequest) -> Result<DumpArtifact, BackupError>;
/// 检查暂存目录是否存在完整归档。
async fn inspect_staging(&self, backup_id: &str)
-> Result<Option<StagedArtifact>, BackupError>;
/// 清理暂存目录中该备份的全部文件。
async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError>;
}
/// 一次 `pg_dump` 请求。
#[derive(Debug, Clone)]
pub struct DumpRequest {
pub backup_id: String,
pub cancellation: gateway_core::engine::CancellationToken,
}
/// 已完成并落盘的归档事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DumpArtifact {
pub path: PathBuf,
pub size_bytes: u64,
pub sha256: String,
}
/// 暂存目录中已存在的完整归档。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StagedArtifact {
pub path: PathBuf,
pub size_bytes: u64,
pub sha256: String,
}
/// S3 兼容对象存储适配器端口。
///
/// 方法接受已经校验的存储配置;SDK client 的构造与缓存属于 adapter 实现细节。
#[async_trait]
pub trait BackupObjectStorePort: Send + Sync {
/// 对专用探针对象执行 Put/Head/Get/Delete。
async fn test_connection(
&self,
config: &BackupStorageConfig,
) -> Result<ConnectionTestResult, BackupError>;
/// 分片流式上传本地归档;取消时中止 multipart 并返回 `backup.cancelled`。
async fn upload_file(
&self,
config: &BackupStorageConfig,
request: UploadObjectRequest,
) -> Result<(), BackupError>;
/// 读取对象大小与 metadata;对象不存在返回 `Ok(None)`。
async fn head_object(
&self,
config: &BackupStorageConfig,
object_key: &str,
) -> Result<Option<BackupObjectMetadata>, BackupError>;
/// 删除对象;对象不存在视为成功。
async fn delete_object(
&self,
config: &BackupStorageConfig,
object_key: &str,
) -> Result<(), BackupError>;
/// 创建短时预签名下载 URL。
async fn presigned_download(
&self,
config: &BackupStorageConfig,
object_key: &str,
file_name: &str,
ttl: Duration,
) -> Result<String, BackupError>;
}
/// 一次对象上传请求。
#[derive(Debug, Clone)]
pub struct UploadObjectRequest {
pub object_key: String,
pub source: PathBuf,
pub metadata: BackupObjectMetadata,
pub cancellation: gateway_core::engine::CancellationToken,
}
/// 备份控制面所需的全部 Store 能力集合。
#[derive(Clone)]
pub struct BackupStorePorts {
repository: Arc<dyn BackupRepository>,
dump: Arc<dyn DatabaseDumpPort>,
object_store: Arc<dyn BackupObjectStorePort>,
}
impl BackupStorePorts {
/// 组合备份端口。
#[must_use]
pub fn new(
repository: Arc<dyn BackupRepository>,
dump: Arc<dyn DatabaseDumpPort>,
object_store: Arc<dyn BackupObjectStorePort>,
) -> Self {
Self {
repository,
dump,
object_store,
}
}
/// 备份任务/配置仓储。
#[must_use]
pub fn repository(&self) -> Arc<dyn BackupRepository> {
self.repository.clone()
}
/// `pg_dump` 导出与暂存。
#[must_use]
pub fn dump(&self) -> Arc<dyn DatabaseDumpPort> {
self.dump.clone()
}
/// S3/R2 对象存储适配器。
#[must_use]
pub fn object_store(&self) -> Arc<dyn BackupObjectStorePort> {
self.object_store.clone()
}
/// 测试与不可用路径使用的空实现;任何调用都返回不可用错误。
#[doc(hidden)]
#[must_use]
pub fn disabled() -> Self {
Self {
repository: Arc::new(UnavailableBackupRepository),
dump: Arc::new(UnavailableDumpPort),
object_store: Arc::new(UnavailableObjectStore),
}
}
}
struct UnavailableBackupRepository;
#[async_trait]
impl BackupRepository for UnavailableBackupRepository {
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
Err(disabled_store())
}
async fn update_storage_settings(
&self,
_command: UpdateBackupStorageCommand,
_context: &MutationContext,
) -> AdminStoreResult<(BackupSettings, Revision)> {
Err(disabled_store())
}
async fn update_schedule_settings(
&self,
_command: UpdateBackupScheduleCommand,
_next_run_at: Option<DateTime<Utc>>,
_context: &MutationContext,
) -> AdminStoreResult<BackupSettings> {
Err(disabled_store())
}
async fn record_verification(
&self,
_storage_revision: u64,
_at: DateTime<Utc>,
) -> AdminStoreResult<bool> {
Err(disabled_store())
}
async fn insert_backup_record(
&self,
_seed: BackupRecordSeed,
) -> AdminStoreResult<BackupRecord> {
Err(disabled_store())
}
async fn insert_scheduled_record(&self, _seed: BackupRecordSeed) -> AdminStoreResult<bool> {
Err(disabled_store())
}
async fn list_backup_records(
&self,
_query: BackupRecordListQuery,
) -> AdminStoreResult<BackupRecordPage> {
Err(disabled_store())
}
async fn load_backup_record(&self, _id: &str) -> AdminStoreResult<Option<BackupRecord>> {
Err(disabled_store())
}
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
Err(disabled_store())
}
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
Err(disabled_store())
}
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
Err(disabled_store())
}
async fn claim_next_queued(
&self,
_now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
Err(disabled_store())
}
async fn transition_status(
&self,
_id: &str,
_from: BackupStatus,
_to: BackupStatus,
_update: StatusTransitionUpdate,
_now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
Err(disabled_store())
}
async fn transition_to_deleting(
&self,
_id: &str,
_now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
Err(disabled_store())
}
async fn delete_record(&self, _id: &str) -> AdminStoreResult<()> {
Err(disabled_store())
}
async fn advance_schedule_cursor(
&self,
_next_run_at: DateTime<Utc>,
_expected_cron: &str,
_expected_timezone: &str,
) -> AdminStoreResult<bool> {
Err(disabled_store())
}
async fn list_scheduled_completed_desc(
&self,
_limit: u32,
) -> AdminStoreResult<Vec<BackupRecord>> {
Err(disabled_store())
}
}
struct UnavailableDumpPort;
#[async_trait]
impl DatabaseDumpPort for UnavailableDumpPort {
async fn dump(&self, _request: DumpRequest) -> Result<DumpArtifact, BackupError> {
Err(disabled_infra())
}
async fn inspect_staging(
&self,
_backup_id: &str,
) -> Result<Option<StagedArtifact>, BackupError> {
Err(disabled_infra())
}
async fn cleanup_staging(&self, _backup_id: &str) -> Result<(), BackupError> {
Err(disabled_infra())
}
}
struct UnavailableObjectStore;
#[async_trait]
impl BackupObjectStorePort for UnavailableObjectStore {
async fn test_connection(
&self,
_config: &BackupStorageConfig,
) -> Result<ConnectionTestResult, BackupError> {
Err(disabled_infra())
}
async fn upload_file(
&self,
_config: &BackupStorageConfig,
_request: UploadObjectRequest,
) -> Result<(), BackupError> {
Err(disabled_infra())
}
async fn head_object(
&self,
_config: &BackupStorageConfig,
_object_key: &str,
) -> Result<Option<BackupObjectMetadata>, BackupError> {
Err(disabled_infra())
}
async fn delete_object(
&self,
_config: &BackupStorageConfig,
_object_key: &str,
) -> Result<(), BackupError> {
Err(disabled_infra())
}
async fn presigned_download(
&self,
_config: &BackupStorageConfig,
_object_key: &str,
_file_name: &str,
_ttl: std::time::Duration,
) -> Result<String, BackupError> {
Err(disabled_infra())
}
}
fn disabled_store() -> crate::ports::store::AdminStoreError {
crate::ports::store::AdminStoreError::new(
crate::ports::store::AdminStoreErrorKind::Unavailable,
"backup",
"backup store is disabled in this configuration",
)
}
fn disabled_infra() -> BackupError {
BackupError::new(
crate::model::backup::code::STORE_UNAVAILABLE,
"backup infrastructure is disabled in this configuration".to_owned(),
)
}
@@ -1,5 +1,6 @@
//! 管理用例依赖的外部能力端口。
pub mod backup;
pub mod provider;
pub mod store;
pub mod system;
@@ -7,6 +7,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use super::backup::BackupStorePorts;
use crate::model::{
MutationContext, Revision,
accounts::{
@@ -300,6 +301,7 @@ pub struct AdminStorePorts {
client_keys: Arc<dyn ClientKeyStore>,
observability: Arc<dyn ObservabilityStore>,
settings: Arc<dyn SettingsStore>,
backup: BackupStorePorts,
}
impl AdminStorePorts {
@@ -310,6 +312,7 @@ impl AdminStorePorts {
client_keys: Arc<dyn ClientKeyStore>,
observability: Arc<dyn ObservabilityStore>,
settings: Arc<dyn SettingsStore>,
backup: BackupStorePorts,
) -> Self {
Self {
accounts,
@@ -317,6 +320,7 @@ impl AdminStorePorts {
client_keys,
observability,
settings,
backup,
}
}
@@ -344,4 +348,9 @@ impl AdminStorePorts {
pub fn settings(&self) -> Arc<dyn SettingsStore> {
self.settings.clone()
}
#[must_use]
pub fn backup(&self) -> BackupStorePorts {
self.backup.clone()
}
}
@@ -217,7 +217,7 @@ impl DefaultAccountsService {
)
.await?;
Ok(AccountDirectoryItem {
status: account_status(&account, now),
status: account_status(&account, quota.limit_reached, now),
usage: usage.into_iter().next(),
account,
quota,
@@ -260,18 +260,37 @@ impl AccountsService for DefaultAccountsService {
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
let account_id = ProviderAccountId::new(account.id.clone())
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
let provider = self
.providers
.require(&account.provider_kind)
.map_err(|error| map_provider_error(error, "provider quota"))?;
provider
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
// 该账号降级为空额度投影,其余账号与页面状态照常返回。
let provider = match self.providers.require(&account.provider_kind) {
Ok(provider) => provider,
Err(error) => {
tracing::warn!(
account_id = %account.id,
error = %error,
"account directory provider is not registered; showing empty quota"
);
return Ok(empty_quota());
}
};
match provider
.quota(ProviderQuotaRequest {
account_id,
refresh: false,
rolling_usage: rolling_usage.get(&account.id).cloned(),
})
.await
.map_err(|error| map_provider_error(error, "provider quota"))
{
Ok(quota) => Ok(quota),
Err(error) => {
tracing::warn!(
account_id = %account.id,
error = %error,
"account directory quota projection failed; showing empty quota"
);
Ok(empty_quota())
}
}
}))
.await
.into_iter()
@@ -283,7 +302,7 @@ impl AccountsService for DefaultAccountsService {
.into_iter()
.zip(quotas)
.map(|(account, quota)| AccountDirectoryItem {
status: account_status(&account, now),
status: account_status(&account, quota.limit_reached, now),
usage: usage.get(&account.id).cloned(),
account,
quota,
@@ -407,7 +426,7 @@ impl AccountsService for DefaultAccountsService {
upstream_model: UpstreamModelId,
) -> Result<AccountConnectionTestEventStream, AdminError> {
let (account, provider) = self.provider_for_account(&account_id).await?;
let initial_status = account_status(&account, Utc::now());
let initial_status = account_status(&account, false, Utc::now());
let model = upstream_model.as_str().to_owned();
let operation = provider
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
@@ -441,7 +460,7 @@ impl AccountsService for DefaultAccountsService {
.ok()
.flatten()
.map_or(initial_status, |account| {
account_status(&account, Utc::now())
account_status(&account, false, Utc::now())
});
match result {
Ok(result) => result
@@ -486,26 +505,47 @@ impl AccountsService for DefaultAccountsService {
}
}
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
fn empty_quota() -> ProviderQuota {
ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
provider_data: None,
}
}
/// 互斥页面状态:availability 终态优先,quota 快照级 `limit_reached` 派生
/// `rate_limited`,`Unknown` 保守映射 `invalid`(不可调度,不显示为 active)。
fn account_status(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatus {
if !account.enabled {
AccountStatus::Disabled
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
AccountStatus::Banned
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
&& account.cooldown_until.is_none_or(|until| until > now))
{
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted {
AccountStatus::QuotaExhausted
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Invalid
) {
AccountStatus::Invalid
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Expired
| crate::model::accounts::AccountAvailability::Invalid
) || account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
{
AccountStatus::Expired
} else if quota_limit_reached {
AccountStatus::RateLimited
} else if account.availability == crate::model::accounts::AccountAvailability::Unknown {
AccountStatus::Invalid
} else {
AccountStatus::Active
}
@@ -0,0 +1,437 @@
//! 备份配置、手动创建、下载与删除用例。
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use gateway_core::routing::snapshot::SnapshotControl;
use secrecy::ExposeSecret as _;
use uuid::Uuid;
use crate::model::{
AdminError, AdminErrorKind, MutationContext,
auth::{AdminAuditEvent, AuditActorKind},
backup::{
BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings,
BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult,
DownloadUrlResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
build_backup_seed, code,
},
};
use crate::ports::{
backup::{BackupObjectStorePort, BackupRepository},
store::AuthStore,
};
use super::map_store_error;
/// 默认下载地址有效期。
const DOWNLOAD_TTL: Duration = Duration::from_secs(5 * 60);
/// API 消费的备份管理服务。
#[async_trait]
pub trait BackupService: Send + Sync {
async fn load_settings(&self) -> Result<BackupSettings, AdminError>;
async fn update_storage(
&self,
context: &MutationContext,
command: UpdateBackupStorageCommand,
) -> Result<BackupSettings, AdminError>;
async fn update_schedule(
&self,
context: &MutationContext,
command: UpdateBackupScheduleCommand,
) -> Result<BackupSettings, AdminError>;
async fn test_connection(
&self,
context: &MutationContext,
) -> Result<ConnectionTestResult, AdminError>;
async fn create_backup(
&self,
context: &MutationContext,
expires_at: Option<DateTime<Utc>>,
) -> Result<BackupRecord, AdminError>;
async fn list_records(
&self,
query: BackupRecordListQuery,
) -> Result<BackupRecordPage, AdminError>;
async fn download_url(
&self,
context: &MutationContext,
backup_id: &str,
) -> Result<DownloadUrlResult, AdminError>;
async fn delete_backup(
&self,
context: &MutationContext,
backup_id: &str,
) -> Result<BackupRecord, AdminError>;
}
pub(crate) struct DefaultBackupService {
repository: Arc<dyn BackupRepository>,
object_store: Arc<dyn BackupObjectStorePort>,
auth: Arc<dyn AuthStore>,
snapshot: Arc<dyn SnapshotControl>,
}
impl DefaultBackupService {
#[must_use]
pub(crate) fn new(
repository: Arc<dyn BackupRepository>,
object_store: Arc<dyn BackupObjectStorePort>,
auth: Arc<dyn AuthStore>,
snapshot: Arc<dyn SnapshotControl>,
) -> Self {
Self {
repository,
object_store,
auth,
snapshot,
}
}
}
#[async_trait]
impl BackupService for DefaultBackupService {
async fn load_settings(&self) -> Result<BackupSettings, AdminError> {
self.repository
.load_settings()
.await
.map_err(|error| map_store_error(error, "backup settings"))
}
async fn update_storage(
&self,
context: &MutationContext,
command: UpdateBackupStorageCommand,
) -> Result<BackupSettings, AdminError> {
validate_storage(&command)?;
let (settings, revision) = self
.repository
.update_storage_settings(command, context)
.await
.map_err(|error| map_store_error(error, "backup storage"))?;
super::publish_committed(self.snapshot.as_ref(), revision).await?;
Ok(settings)
}
async fn update_schedule(
&self,
context: &MutationContext,
command: UpdateBackupScheduleCommand,
) -> Result<BackupSettings, AdminError> {
validate_schedule(&command)?;
let next_run_at = if command.schedule_enabled {
Some(
crate::backup::policy::BackupSchedule::parse(
&command.cron_expression,
&command.schedule_timezone,
)
.map_err(map_backup_error)?
.next_after(Utc::now())
.ok_or_else(|| AdminError::bad_request("无法从计划推导下次执行时间"))?,
)
} else {
None
};
self.repository
.update_schedule_settings(command, next_run_at, context)
.await
.map_err(|error| map_store_error(error, "backup schedule"))
}
async fn test_connection(
&self,
context: &MutationContext,
) -> Result<ConnectionTestResult, AdminError> {
let settings = self
.repository
.load_settings()
.await
.map_err(|error| map_store_error(error, "backup settings"))?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Ok(ConnectionTestResult {
ok: false,
stage: "putObject",
code: Some(code::INVALID_CONFIG),
message: "备份存储配置不完整,无法执行连接测试".to_owned(),
});
};
let result = self
.object_store
.test_connection(&config)
.await
.map_err(map_backup_error)?;
if result.ok {
let _ = self
.repository
.record_verification(config.storage_revision, Utc::now())
.await
.map_err(|error| map_store_error(error, "backup settings"))?;
}
self.append_audit(
context,
"backup.s3_connection_tested",
"backup_settings",
vec![],
)
.await?;
Ok(result)
}
async fn create_backup(
&self,
context: &MutationContext,
expires_at: Option<DateTime<Utc>>,
) -> Result<BackupRecord, AdminError> {
let settings = self
.repository
.load_settings()
.await
.map_err(|error| map_store_error(error, "backup settings"))?;
if !settings.storage_configured() {
return Err(AdminError::bad_request(
"备份存储配置不完整,请先保存存储配置",
));
}
if !settings.storage_verified() {
return Err(AdminError::bad_request(
"备份存储尚未通过连接测试,请先测试存储配置",
));
}
let prefix = settings.prefix.as_deref().unwrap_or_default();
let seed = build_backup_seed(
BackupTriggerKind::Manual,
None,
prefix,
Utc::now(),
expires_at,
)
.map_err(map_backup_error)?;
let record = self
.repository
.insert_backup_record(seed)
.await
.map_err(|error| map_store_error(error, "backup record"))?;
self.append_audit(context, "backup.created", "backup", vec![record.id.clone()])
.await?;
Ok(record)
}
async fn list_records(
&self,
query: BackupRecordListQuery,
) -> Result<BackupRecordPage, AdminError> {
self.repository
.list_backup_records(query)
.await
.map_err(|error| map_store_error(error, "backup records"))
}
async fn download_url(
&self,
context: &MutationContext,
backup_id: &str,
) -> Result<DownloadUrlResult, AdminError> {
let record = self.require_record(backup_id).await?;
if record.status != BackupStatus::Completed {
return Err(AdminError::conflict(
"只有 completed 的备份才能创建下载地址",
));
}
let settings = self
.repository
.load_settings()
.await
.map_err(|error| map_store_error(error, "backup settings"))?;
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
return Err(AdminError::bad_request("备份存储配置不完整"));
};
let file_name = crate::model::backup::build_download_file_name(backup_id);
let url = self
.object_store
.presigned_download(&config, &record.object_key, &file_name, DOWNLOAD_TTL)
.await
.map_err(map_backup_error)?;
self.append_audit(
context,
"backup.download_url_created",
"backup",
vec![backup_id.to_owned()],
)
.await?;
Ok(DownloadUrlResult {
url,
file_name,
expires_in: DOWNLOAD_TTL,
})
}
async fn delete_backup(
&self,
context: &MutationContext,
backup_id: &str,
) -> Result<BackupRecord, AdminError> {
let record = self.require_record(backup_id).await?;
if !record.status.can_be_deleted() {
return Err(AdminError::conflict(
"只有 completed 或 failed 的备份才能删除",
));
}
let now = Utc::now();
let deleting = self
.repository
.transition_to_deleting(backup_id, now)
.await
.map_err(|error| map_store_error(error, "backup record"))?
.ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?;
self.append_audit(
context,
"backup.delete_requested",
"backup",
vec![backup_id.to_owned()],
)
.await?;
Ok(deleting)
}
}
impl DefaultBackupService {
async fn require_record(&self, backup_id: &str) -> Result<BackupRecord, AdminError> {
self.repository
.load_backup_record(backup_id)
.await
.map_err(|error| map_store_error(error, "backup record"))?
.ok_or_else(|| AdminError::not_found("备份记录不存在"))
}
async fn append_audit(
&self,
context: &MutationContext,
action: &str,
entity_ref: &str,
changed_fields: Vec<String>,
) -> Result<(), AdminError> {
let actor_kind = match context.actor {
crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession,
crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey,
crate::model::MutationActor::System => AuditActorKind::System,
};
let actor_ref = match &context.actor {
crate::model::MutationActor::AdminSession { admin_user_id } => admin_user_id.clone(),
crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(),
crate::model::MutationActor::System => "system".to_owned(),
};
let actor_admin_user_id = match &context.actor {
crate::model::MutationActor::AdminSession { admin_user_id } => {
Some(admin_user_id.clone())
}
_ => None,
};
let event = AdminAuditEvent {
id: format!("audit_{}", Uuid::now_v7().simple()),
actor_kind,
actor_admin_user_id,
actor_ref,
request_id: Some(context.request_id.clone()),
action: action.to_owned(),
entity_kind: "backup".to_owned(),
entity_ref: entity_ref.to_owned(),
config_revision: None,
changed_fields,
occurred_at: Utc::now(),
};
self.auth
.append_audit_event(event)
.await
.map_err(|error| map_store_error(error, "backup audit"))
}
}
/// 校验存储配置;空串视为非法,Secret 缺省表示保留旧值。
fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminError> {
validate_endpoint(&command.endpoint)?;
validate_nonempty("region", &command.region)?;
validate_nonempty("bucket", &command.bucket)?;
validate_nonempty("accessKeyId", &command.access_key_id)?;
if let Some(secret) = command.secret_access_key.as_ref() {
let value = secret.expose_secret();
if value.is_empty() {
return Err(AdminError::bad_request(
"secretAccessKey 不能为空字符串;缺省表示保留旧值",
));
}
validate_length("secretAccessKey", value, 512)?;
}
crate::model::backup::normalize_prefix(&command.prefix).map_err(map_backup_error)?;
Ok(())
}
fn validate_endpoint(value: &str) -> Result<(), AdminError> {
if value.is_empty() {
return Err(AdminError::bad_request("endpoint 不能为空"));
}
if !value.starts_with("https://") {
return Err(AdminError::bad_request(
"endpoint 必须使用 HTTPS;仅测试或显式开发配置允许 HTTP",
));
}
if value.contains('@') || value.bytes().any(|byte| byte.is_ascii_control()) {
return Err(AdminError::bad_request("endpoint 格式非法"));
}
validate_length("endpoint", value, 2048)
}
fn validate_nonempty(field: &str, value: &str) -> Result<(), AdminError> {
if value.is_empty() {
return Err(AdminError::bad_request(format!("{field} 不能为空")));
}
validate_length(field, value, 255)
}
fn validate_length(field: &str, value: &str, max: usize) -> Result<(), AdminError> {
if value.len() > max || value.bytes().any(|byte| byte.is_ascii_control()) {
return Err(AdminError::bad_request(format!(
"{field} 超出长度或包含非法字符"
)));
}
Ok(())
}
/// 校验调度配置并解析 Cron 与时区。
fn validate_schedule(command: &UpdateBackupScheduleCommand) -> Result<(), AdminError> {
if command.cron_expression.trim().is_empty() {
return Err(AdminError::bad_request("cronExpression 不能为空"));
}
crate::backup::policy::BackupSchedule::parse(
&command.cron_expression,
&command.schedule_timezone,
)
.map_err(map_backup_error)?;
Ok(())
}
/// 备份基础设施错误到既有 `AdminErrorKind` 的唯一边界映射。
fn map_backup_error(error: BackupError) -> AdminError {
match error.code() {
code::INVALID_CONFIG | code::INVALID_CRON | code::INVALID_TIMEZONE => {
AdminError::bad_request(error.message().to_owned())
}
code::RECORD_NOT_FOUND => AdminError::not_found(error.message().to_owned()),
code::ACTIVE_TASK_CONFLICT | code::STATE_CONFLICT | code::STORAGE_IDENTITY_LOCKED => {
AdminError::conflict(error.message().to_owned())
}
code::STORE_UNAVAILABLE => {
AdminError::new(AdminErrorKind::Unavailable, error.message().to_owned())
}
_ => AdminError::bad_gateway(error.message().to_owned()),
}
}
@@ -2,6 +2,7 @@
pub mod accounts;
pub mod auth;
pub mod backup;
pub mod client_keys;
pub mod observability;
pub mod openai;
@@ -247,6 +247,7 @@ impl ObservabilityService for DefaultObservabilityService {
items: items
.into_iter()
.map(|item| DiagnosticsItem {
key: item.key,
name: item.name,
request_count: item.request_count,
success_count: item.request_count.saturating_sub(item.failure_count),
@@ -254,6 +255,7 @@ impl ObservabilityService for DefaultObservabilityService {
error_rate: rate_or_zero(item.failure_count, item.request_count),
request_share: rate_or_zero(item.request_count, total_requests),
average_latency_ms: item.average_latency_ms,
latency_p95_ms: item.latency_p95_ms,
estimated_cost: usd_cost(&item.costs),
attempt_count: item.attempt_count,
total_tokens: item.total_tokens,
@@ -492,14 +494,14 @@ impl DefaultObservabilityService {
}
}
/// 逐条尽力把 calculated 总额升级为完整分解;单条脏数据(非法 Provider kind
/// 或费用规则失败)只保留该条已存的总额,不影响整页返回。
/// 逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回。
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
for record in records {
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
continue;
};
if source != "calculated" {
if !matches!(source.as_str(), "calculated" | "provider_reported") {
continue;
}
let (Some(provider), Some(upstream_model_id)) = (
@@ -0,0 +1,5 @@
//! 备份模块测试:镜像 src/backup/ 的 policy 与 task。
mod policy;
pub mod support;
mod task;
@@ -0,0 +1,171 @@
//! 备份调度与保留策略(backup/policy.rs)的纯逻辑测试。
use chrono::{Duration, TimeZone as _, Utc};
use gateway_admin::{
backup::policy::{BackupSchedule, RetentionReason, decide_retention},
model::backup::{BackupRecord, BackupStatus, BackupTriggerKind},
};
fn record(
id: &str,
trigger: BackupTriggerKind,
status: BackupStatus,
created_at: chrono::DateTime<Utc>,
completed_at: Option<chrono::DateTime<Utc>>,
) -> BackupRecord {
BackupRecord {
id: id.to_owned(),
trigger_kind: trigger,
status,
scheduled_at: if trigger == BackupTriggerKind::Scheduled {
Some(created_at)
} else {
None
},
object_key: format!("k/{id}"),
size_bytes: Some(1),
sha256: Some("a".repeat(64)),
attempt_count: 0,
error_code: None,
error_message: None,
started_at: Some(created_at),
completed_at,
expires_at: None,
created_at,
updated_at: created_at,
}
}
#[test]
fn parse_rejects_bad_cron_and_timezone() {
assert!(BackupSchedule::parse("61 * * * *", "Asia/Shanghai").is_err());
assert!(BackupSchedule::parse("0 2 * * *", "Not/AZone").is_err());
assert!(BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").is_ok());
}
#[test]
fn next_after_is_strictly_later() {
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
let at = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
let next = schedule.next_after(at).unwrap();
assert_eq!(next, Utc.with_ymd_and_hms(2026, 8, 2, 18, 0, 0).unwrap());
}
#[test]
fn last_firing_is_at_or_before() {
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
// 北京时间 02:00 = 前一日 18:00 UTC。
let before = Utc.with_ymd_and_hms(2026, 8, 1, 17, 59, 0).unwrap();
assert_eq!(
schedule.last_firing_at_or_before(before).unwrap(),
Utc.with_ymd_and_hms(2026, 7, 31, 18, 0, 0).unwrap()
);
let exact = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
assert_eq!(schedule.last_firing_at_or_before(exact).unwrap(), exact);
let after = Utc.with_ymd_and_hms(2026, 8, 1, 19, 0, 0).unwrap();
assert_eq!(schedule.last_firing_at_or_before(after).unwrap(), exact);
}
#[test]
fn dst_spring_forward_skips_missing_hour() {
let schedule = BackupSchedule::parse("30 2 * * *", "America/New_York").unwrap();
let at = Utc.with_ymd_and_hms(2026, 3, 8, 6, 29, 0).unwrap(); // 01:29 EST
let next = schedule.next_after(at).unwrap();
assert_eq!(next, Utc.with_ymd_and_hms(2026, 3, 9, 6, 30, 0).unwrap());
let at_after_spring = Utc.with_ymd_and_hms(2026, 3, 8, 10, 0, 0).unwrap(); // 06:00 EDT
let last = schedule.last_firing_at_or_before(at_after_spring).unwrap();
assert_eq!(last, Utc.with_ymd_and_hms(2026, 3, 7, 7, 30, 0).unwrap());
}
#[test]
fn retention_count_keeps_newest_n() {
let now = Utc::now();
let base = now - Duration::days(10);
let records = vec![
record(
"newest",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
base + Duration::days(2),
Some(base + Duration::days(2)),
),
record(
"middle",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
base + Duration::days(1),
Some(base + Duration::days(1)),
),
record(
"oldest",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
base,
Some(base),
),
];
let decisions = decide_retention(0, 2, now, &records);
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
assert_eq!(ids, vec!["oldest"]);
assert_eq!(decisions[0].reason, RetentionReason::Count);
}
#[test]
fn retention_days_deletes_old_scheduled_but_keeps_newest() {
let now = Utc::now();
let records = vec![
record(
"recent",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
now - Duration::days(1),
Some(now - Duration::days(1)),
),
record(
"ancient",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
now - Duration::days(40),
Some(now - Duration::days(40)),
),
];
let decisions = decide_retention(30, 0, now, &records);
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
assert_eq!(ids, vec!["ancient"]);
}
#[test]
fn newest_scheduled_never_auto_deleted_even_when_ancient() {
let now = Utc::now();
let records = vec![record(
"sole",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
now - Duration::days(400),
Some(now - Duration::days(400)),
)];
assert!(decide_retention(30, 1, now, &records).is_empty());
}
#[test]
fn disabled_thresholds_delete_nothing() {
let now = Utc::now();
let records = vec![
record(
"newest",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
now - Duration::days(2),
Some(now - Duration::days(2)),
),
record(
"old",
BackupTriggerKind::Scheduled,
BackupStatus::Completed,
now - Duration::days(90),
Some(now - Duration::days(90)),
),
];
assert!(decide_retention(0, 0, now, &records).is_empty());
}
@@ -0,0 +1,652 @@
//! 备份测试共享 fixture:内存 fake 端口与辅助函数。
//!
//! 用内存 fake 端口驱动编排层,不依赖 MinIO/真实 S3。
//! s3.rs / pg_dump.rs 是 SDK 薄适配器,协议正确性交给 aws-sdk / pg_dump 保证。
//! 供 backup::task 与 use_case::backup 的测试复用。
use std::sync::Mutex;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use gateway_admin::{
model::{
MutationActor, MutationContext, Revision,
auth::AdminAuditEvent,
backup::{
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
},
},
ports::{
backup::{
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpArtifact, DumpRequest,
StagedArtifact, StatusTransitionUpdate, UploadObjectRequest,
},
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult, AuthStore},
},
};
use secrecy::SecretString;
use sha2::{Digest as _, Sha256};
use tempfile::TempDir;
/// 假归档内容;dump 与上传校验共用同一字节与校验值。
pub(crate) const DUMP_CONTENT: &[u8] = b"custom-format-archive-bytes-0123456789";
/// 计算一段内容的 SHA-256 十六进制。
pub(crate) fn sha256_hex(content: &[u8]) -> String {
hex::encode(Sha256::digest(content))
}
/// 返回 32 位十六进制的合法备份记录 id。
pub(crate) fn backup_id(suffix: &str) -> String {
format!("backup_{}", sha256_hex(suffix.as_bytes()))
}
/// 已配置且已通过连接测试的设置快照。
pub(crate) fn configured_settings() -> BackupSettings {
let now = Utc::now();
BackupSettings {
storage_revision: 1,
endpoint: Some("https://s3.example.com".to_owned()),
region: Some("auto".to_owned()),
bucket: Some("backup-bucket".to_owned()),
access_key_id: Some("ak".to_owned()),
secret_access_key: Some(SecretString::from("sk-value")),
prefix: Some("codex".to_owned()),
force_path_style: false,
schedule_enabled: false,
cron_expression: None,
schedule_timezone: None,
retention_days: 0,
retention_count: 0,
next_run_at: None,
last_verified_at: Some(now),
updated_at: now,
}
}
/// 从 seed 构造 queued 记录。
pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord {
let now = Utc::now();
BackupRecord {
id: seed.id.clone(),
trigger_kind: seed.trigger_kind,
status: BackupStatus::Queued,
scheduled_at: seed.scheduled_at,
object_key: seed.object_key.clone(),
size_bytes: None,
sha256: None,
attempt_count: 0,
error_code: None,
error_message: None,
started_at: None,
completed_at: None,
expires_at: seed.expires_at,
created_at: now,
updated_at: now,
}
}
/// 内存版备份仓储。
pub(crate) struct FakeBackupRepository {
settings: Mutex<BackupSettings>,
records: Mutex<Vec<BackupRecord>>,
}
impl FakeBackupRepository {
pub(crate) fn new(settings: BackupSettings) -> Self {
Self {
settings: Mutex::new(settings),
records: Mutex::new(Vec::new()),
}
}
/// 测试辅助:读取当前全部记录。
pub(crate) fn all_records(&self) -> Vec<BackupRecord> {
self.records.lock().expect("records").clone()
}
/// 测试辅助:强制把一条记录标记为指定时间完成的 completed。
pub(crate) fn set_completed(&self, id: &str, completed_at: DateTime<Utc>) {
let mut records = self.records.lock().expect("records");
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
record.status = BackupStatus::Completed;
record.completed_at = Some(completed_at);
record.updated_at = Utc::now();
}
}
/// 测试辅助:强制把一条记录迁移到指定状态。
pub(crate) fn force_status(&self, id: &str, status: BackupStatus) {
let mut records = self.records.lock().expect("records");
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
record.status = status;
if matches!(status, BackupStatus::Completed | BackupStatus::Failed) {
record.completed_at = Some(Utc::now());
}
record.updated_at = Utc::now();
}
}
}
#[async_trait]
impl BackupRepository for FakeBackupRepository {
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
Ok(self.settings.lock().expect("settings").clone())
}
async fn update_storage_settings(
&self,
command: UpdateBackupStorageCommand,
_context: &MutationContext,
) -> AdminStoreResult<(BackupSettings, Revision)> {
let mut settings = self.settings.lock().expect("settings");
settings.storage_revision += 1;
settings.endpoint = Some(command.endpoint);
settings.region = Some(command.region);
settings.bucket = Some(command.bucket);
settings.access_key_id = Some(command.access_key_id);
if let Some(secret) = command.secret_access_key {
settings.secret_access_key = Some(secret);
}
settings.prefix = Some(command.prefix);
settings.force_path_style = command.force_path_style;
settings.last_verified_at = None;
settings.updated_at = Utc::now();
let revision = Revision::new(settings.storage_revision).map_err(|_| {
AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", "zero revision")
})?;
Ok((settings.clone(), revision))
}
async fn update_schedule_settings(
&self,
command: UpdateBackupScheduleCommand,
next_run_at: Option<DateTime<Utc>>,
_context: &MutationContext,
) -> AdminStoreResult<BackupSettings> {
let mut settings = self.settings.lock().expect("settings");
settings.schedule_enabled = command.schedule_enabled;
settings.cron_expression = Some(command.cron_expression);
settings.schedule_timezone = Some(command.schedule_timezone);
settings.retention_days = command.retention_days;
settings.retention_count = command.retention_count;
settings.next_run_at = next_run_at;
settings.updated_at = Utc::now();
Ok(settings.clone())
}
async fn record_verification(
&self,
storage_revision: u64,
at: DateTime<Utc>,
) -> AdminStoreResult<bool> {
let mut settings = self.settings.lock().expect("settings");
if settings.storage_revision != storage_revision {
return Ok(false);
}
settings.last_verified_at = Some(at);
Ok(true)
}
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
let mut records = self.records.lock().expect("records");
if records.iter().any(|record| record.status.is_active()) {
return Err(AdminStoreError::new(
AdminStoreErrorKind::Conflict,
"backup record",
"an active backup task already exists",
));
}
let record = queued_record(&seed);
records.push(record.clone());
Ok(record)
}
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool> {
let mut records = self.records.lock().expect("records");
if records.iter().any(|record| record.status.is_active())
|| records.iter().any(|record| {
record.trigger_kind == BackupTriggerKind::Scheduled
&& record.scheduled_at == seed.scheduled_at
})
{
return Ok(false);
}
records.push(queued_record(&seed));
Ok(true)
}
async fn list_backup_records(
&self,
query: BackupRecordListQuery,
) -> AdminStoreResult<BackupRecordPage> {
let records = self.records.lock().expect("records");
let mut items = records
.iter()
.filter(|record| {
query.status.is_none_or(|status| record.status == status)
&& query
.trigger
.is_none_or(|trigger| record.trigger_kind == trigger)
})
.cloned()
.collect::<Vec<_>>();
items.sort_by(|a, b| {
b.created_at
.cmp(&a.created_at)
.then_with(|| b.id.cmp(&a.id))
});
let total = items.len() as u64;
let offset =
(u64::from(query.page.saturating_sub(1)) * u64::from(query.page_size.get())) as usize;
let limit = usize::from(query.page_size.get());
items = items.into_iter().skip(offset).take(limit).collect();
Ok(BackupRecordPage {
items,
total,
page: query.page,
page_size: query.page_size,
})
}
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>> {
Ok(self
.records
.lock()
.expect("records")
.iter()
.find(|record| record.id == id)
.cloned())
}
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
Ok(self
.records
.lock()
.expect("records")
.iter()
.filter(|record| {
matches!(
record.status,
BackupStatus::Dumping | BackupStatus::Uploading
)
})
.cloned()
.collect())
}
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
Ok(self
.records
.lock()
.expect("records")
.iter()
.filter(|record| record.status == BackupStatus::Deleting)
.cloned()
.collect())
}
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
let now = Utc::now();
Ok(self
.records
.lock()
.expect("records")
.iter()
.filter(|record| {
record
.expires_at
.is_some_and(|expires_at| expires_at <= now)
&& matches!(
record.status,
BackupStatus::Completed | BackupStatus::Failed
)
})
.cloned()
.collect())
}
async fn claim_next_queued(
&self,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
let mut records = self.records.lock().expect("records");
let Some(record) = records
.iter_mut()
.find(|record| record.status == BackupStatus::Queued)
else {
return Ok(None);
};
record.status = BackupStatus::Dumping;
record.started_at = Some(now);
record.attempt_count += 1;
record.updated_at = now;
Ok(Some(record.clone()))
}
async fn transition_status(
&self,
id: &str,
from: BackupStatus,
to: BackupStatus,
update: StatusTransitionUpdate,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
let mut records = self.records.lock().expect("records");
let Some(record) = records
.iter_mut()
.find(|record| record.id == id && record.status == from)
else {
return Ok(None);
};
record.status = to;
if let Some(size) = update.size_bytes {
record.size_bytes = Some(size);
}
if let Some(sha256) = update.sha256 {
record.sha256 = Some(sha256);
}
if let Some(code) = update.error_code {
record.error_code = Some(code);
}
if let Some(message) = update.error_message {
record.error_message = Some(message);
}
record.completed_at = update.completed_at.or_else(|| {
matches!(to, BackupStatus::Completed | BackupStatus::Failed).then_some(now)
});
record.updated_at = now;
Ok(Some(record.clone()))
}
async fn transition_to_deleting(
&self,
id: &str,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
let mut records = self.records.lock().expect("records");
let Some(record) = records
.iter_mut()
.find(|record| record.id == id && record.status.can_be_deleted())
else {
return Ok(None);
};
record.status = BackupStatus::Deleting;
record.updated_at = now;
Ok(Some(record.clone()))
}
async fn delete_record(&self, id: &str) -> AdminStoreResult<()> {
let mut records = self.records.lock().expect("records");
records.retain(|record| record.id != id);
Ok(())
}
async fn advance_schedule_cursor(
&self,
next_run_at: DateTime<Utc>,
_expected_cron: &str,
_expected_timezone: &str,
) -> AdminStoreResult<bool> {
let mut settings = self.settings.lock().expect("settings");
settings.next_run_at = Some(next_run_at);
Ok(true)
}
async fn list_scheduled_completed_desc(
&self,
_limit: u32,
) -> AdminStoreResult<Vec<BackupRecord>> {
let mut records = self
.records
.lock()
.expect("records")
.iter()
.filter(|record| {
record.trigger_kind == BackupTriggerKind::Scheduled
&& record.status == BackupStatus::Completed
})
.cloned()
.collect::<Vec<_>>();
records.sort_by(|a, b| {
b.completed_at
.cmp(&a.completed_at)
.then_with(|| b.id.cmp(&a.id))
});
Ok(records)
}
}
/// 假 `pg_dump` 端口:把固定内容写入临时文件作为归档;可注入失败。
pub(crate) struct FakeDumpPort {
dir: TempDir,
fail_dump: bool,
}
impl FakeDumpPort {
pub(crate) fn new() -> Self {
Self {
dir: tempfile::tempdir().expect("temp dir"),
fail_dump: false,
}
}
/// 让下一次 `dump` 返回 `backup.pg_dump_failed`。
pub(crate) fn fail_next_dump(mut self) -> Self {
self.fail_dump = true;
self
}
fn artifact(&self, backup_id: &str) -> DumpArtifact {
let path = self.dir.path().join(format!("{backup_id}.dump"));
std::fs::write(&path, DUMP_CONTENT).expect("write artifact");
DumpArtifact {
path,
size_bytes: DUMP_CONTENT.len() as u64,
sha256: sha256_hex(DUMP_CONTENT),
}
}
}
#[async_trait]
impl DatabaseDumpPort for FakeDumpPort {
async fn dump(
&self,
request: DumpRequest,
) -> Result<DumpArtifact, gateway_admin::model::backup::BackupError> {
if self.fail_dump {
return Err(gateway_admin::model::backup::BackupError::new(
gateway_admin::model::backup::code::PG_DUMP_FAILED,
"pg_dump failed".to_owned(),
));
}
if request.cancellation.is_cancelled() {
return Err(gateway_admin::model::backup::BackupError::new(
gateway_admin::model::backup::code::CANCELLED,
"cancelled".to_owned(),
));
}
Ok(self.artifact(&request.backup_id))
}
async fn inspect_staging(
&self,
backup_id: &str,
) -> Result<Option<StagedArtifact>, gateway_admin::model::backup::BackupError> {
let path = self.dir.path().join(format!("{backup_id}.dump"));
if !path.exists() {
return Ok(None);
}
Ok(Some(StagedArtifact {
path,
size_bytes: DUMP_CONTENT.len() as u64,
sha256: sha256_hex(DUMP_CONTENT),
}))
}
async fn cleanup_staging(
&self,
_backup_id: &str,
) -> Result<(), gateway_admin::model::backup::BackupError> {
Ok(())
}
}
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
pub(crate) struct FakeObjectStore {
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
}
impl FakeObjectStore {
pub(crate) fn new() -> Self {
Self {
objects: Mutex::new(std::collections::HashMap::new()),
}
}
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
self.objects.lock().expect("objects").get(key).cloned()
}
}
#[async_trait]
impl BackupObjectStorePort for FakeObjectStore {
async fn test_connection(
&self,
_config: &BackupStorageConfig,
) -> Result<ConnectionTestResult, gateway_admin::model::backup::BackupError> {
Ok(ConnectionTestResult {
ok: true,
stage: "deleteObject",
code: None,
message: "连接测试成功".to_owned(),
})
}
async fn upload_file(
&self,
_config: &BackupStorageConfig,
request: UploadObjectRequest,
) -> Result<(), gateway_admin::model::backup::BackupError> {
let content = std::fs::read(&request.source).map_err(|_| {
gateway_admin::model::backup::BackupError::new(
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
"read source".to_owned(),
)
})?;
self.objects
.lock()
.expect("objects")
.insert(request.object_key, content);
Ok(())
}
async fn head_object(
&self,
_config: &BackupStorageConfig,
object_key: &str,
) -> Result<Option<BackupObjectMetadata>, gateway_admin::model::backup::BackupError> {
let Some(content) = self.object(object_key) else {
return Ok(None);
};
Ok(Some(BackupObjectMetadata::new(
"backup-test".to_owned(),
sha256_hex(&content),
Utc::now(),
content.len() as u64,
)))
}
async fn delete_object(
&self,
_config: &BackupStorageConfig,
object_key: &str,
) -> Result<(), gateway_admin::model::backup::BackupError> {
self.objects.lock().expect("objects").remove(object_key);
Ok(())
}
async fn presigned_download(
&self,
_config: &BackupStorageConfig,
object_key: &str,
_file_name: &str,
_ttl: std::time::Duration,
) -> Result<String, gateway_admin::model::backup::BackupError> {
Ok(format!("https://presigned.example.com/{object_key}"))
}
}
/// 记录审计事件的假 AuthStore。
pub(crate) struct FakeAuthStore {
audit: Mutex<Vec<AdminAuditEvent>>,
}
impl FakeAuthStore {
pub(crate) fn new() -> Self {
Self {
audit: Mutex::new(Vec::new()),
}
}
pub(crate) fn audit_actions(&self) -> Vec<String> {
self.audit
.lock()
.expect("audit")
.iter()
.map(|event| event.action.clone())
.collect()
}
}
#[async_trait]
impl AuthStore for FakeAuthStore {
async fn load_password_hash(&self, _admin_user_id: &str) -> AdminStoreResult<Option<String>> {
Ok(None)
}
async fn create_password_hash_if_absent(
&self,
_admin_user_id: &str,
_password_hash: &str,
) -> AdminStoreResult<bool> {
Ok(true)
}
async fn load_admin_api_key(
&self,
) -> AdminStoreResult<Option<gateway_admin::model::settings::AdminApiKey>> {
Ok(None)
}
async fn load_session(
&self,
_session_id: &str,
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
Ok(None)
}
async fn store_session(
&self,
_session_id: &str,
_session: &gateway_admin::model::auth::AdminSession,
) -> AdminStoreResult<()> {
Ok(())
}
async fn delete_session(
&self,
_session_id: &str,
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
Ok(None)
}
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
self.audit.lock().expect("audit").push(event);
Ok(())
}
}
/// 系统发起者上下文。
pub(crate) fn system_context() -> MutationContext {
MutationContext {
actor: MutationActor::System,
request_id: "backup-runtime-test".to_owned(),
}
}
@@ -0,0 +1,165 @@
//! BackupTask 执行链测试:驱动内存 fake 端口跑完整 dump/upload/verify/retention 流程。
use std::sync::Arc;
use chrono::{Duration, Utc};
use gateway_admin::{
backup::task::BackupTask,
model::backup::{
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
},
ports::backup::BackupRepository,
};
use gateway_core::engine::CancellationToken;
use super::support::{
DUMP_CONTENT, FakeBackupRepository, FakeDumpPort, FakeObjectStore, backup_id,
configured_settings, sha256_hex, system_context,
};
#[tokio::test]
async fn backup_task_runs_full_dump_upload_verify_pipeline() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store = Arc::new(FakeObjectStore::new());
let seed = BackupRecordSeed {
id: backup_id("manual"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/manual.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
task.run_cycle(&CancellationToken::new())
.await
.expect("run cycle");
let records = repository.all_records();
assert_eq!(records.len(), 1);
let finished = &records[0];
assert_eq!(finished.status, BackupStatus::Completed);
assert_eq!(finished.size_bytes, Some(DUMP_CONTENT.len() as u64));
assert_eq!(
finished.sha256.as_deref(),
Some(sha256_hex(DUMP_CONTENT).as_str())
);
// 对象已上传且内容与本地归档一致。
let stored = object_store
.object(&seed.object_key)
.expect("uploaded object exists");
assert_eq!(stored.as_slice(), DUMP_CONTENT);
}
#[tokio::test]
async fn backup_task_marks_failed_when_dump_port_fails() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new().fail_next_dump());
let object_store = Arc::new(FakeObjectStore::new());
let seed = BackupRecordSeed {
id: backup_id("fail"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/fail.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
task.run_cycle(&CancellationToken::new())
.await
.expect("run cycle");
let records = repository.all_records();
let failed = &records[0];
assert_eq!(failed.status, BackupStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some("backup.pg_dump_failed"));
assert!(failed.completed_at.is_some());
// 归档未上传,对象不存在。
assert!(object_store.object(&seed.object_key).is_none());
}
#[tokio::test]
async fn retention_cleans_expired_scheduled_backups() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store = Arc::new(FakeObjectStore::new());
// 造一条旧 completed 计划备份(超过 retention_days)与一条新备份。
let old_seed = BackupRecordSeed {
id: backup_id("old"),
trigger_kind: BackupTriggerKind::Scheduled,
scheduled_at: Some(Utc::now() - Duration::days(40)),
object_key: "codex/2026/06/01/old.dump".to_owned(),
expires_at: None,
};
let new_seed = BackupRecordSeed {
id: backup_id("new"),
trigger_kind: BackupTriggerKind::Scheduled,
scheduled_at: Some(Utc::now() - Duration::days(1)),
object_key: "codex/2026/07/01/new.dump".to_owned(),
expires_at: None,
};
repository
.insert_scheduled_record(old_seed.clone())
.await
.expect("insert old");
// 旧备份 40 天前完成(释放活跃名额),再插入新备份。
let now = Utc::now();
repository.set_completed(&backup_id("old"), now - Duration::days(40));
repository
.insert_scheduled_record(new_seed.clone())
.await
.expect("insert new");
// 新备份 1 天前完成。
repository.set_completed(&backup_id("new"), now - Duration::days(1));
// 两个对象都已上传。
object_store
.objects
.lock()
.expect("objects")
.insert(old_seed.object_key.clone(), DUMP_CONTENT.to_vec());
object_store
.objects
.lock()
.expect("objects")
.insert(new_seed.object_key.clone(), DUMP_CONTENT.to_vec());
// 设置 retention_days = 30。
repository
.update_schedule_settings(
UpdateBackupScheduleCommand {
schedule_enabled: false,
cron_expression: "0 2 * * *".to_owned(),
schedule_timezone: "Asia/Shanghai".to_owned(),
retention_days: 30,
retention_count: 0,
},
None,
&system_context(),
)
.await
.expect("set retention");
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
task.run_cycle(&CancellationToken::new())
.await
.expect("run cycle");
// 旧备份被硬删除,对象被删除;新备份保留。
let records = repository.all_records();
let ids: Vec<_> = records.iter().map(|record| record.id.as_str()).collect();
assert_eq!(ids, vec![backup_id("new").as_str()]);
assert!(object_store.object(&old_seed.object_key).is_none());
assert!(object_store.object(&new_seed.object_key).is_some());
}
@@ -17,6 +17,8 @@ use gateway_admin::{
ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore},
};
mod backup;
mod model;
mod use_case;
#[test]
@@ -125,6 +127,7 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
quota_window("shortTerm", Some(18_000), Some(30.0)),
quota_window("shortTerm", Some(14_400), Some(45.0)),
],
limit_reached: false,
provider_data: None,
};
@@ -144,6 +147,7 @@ fn quota_window(
window_seconds,
used_percent,
reset_at: None,
limit_reached: false,
local_usage: None,
provider_data: None,
}
@@ -0,0 +1,104 @@
//! 备份领域模型(model/backup.rs)的纯逻辑测试。
use chrono::{TimeZone as _, Utc};
use secrecy::SecretString;
use gateway_admin::model::backup::{
BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, build_object_key,
};
#[test]
fn status_machine_rejects_invalid_transitions() {
assert!(BackupStatus::Queued.allows_transition_to(BackupStatus::Dumping));
assert!(BackupStatus::Dumping.allows_transition_to(BackupStatus::Uploading));
assert!(BackupStatus::Uploading.allows_transition_to(BackupStatus::Completed));
assert!(BackupStatus::Completed.allows_transition_to(BackupStatus::Deleting));
assert!(!BackupStatus::Queued.allows_transition_to(BackupStatus::Completed));
assert!(!BackupStatus::Completed.allows_transition_to(BackupStatus::Dumping));
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Failed));
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Completed));
}
#[test]
fn status_classification() {
assert!(BackupStatus::Queued.is_active());
assert!(BackupStatus::Uploading.is_active());
assert!(!BackupStatus::Completed.is_active());
assert!(BackupStatus::Completed.can_be_deleted());
assert!(BackupStatus::Failed.can_be_deleted());
assert!(!BackupStatus::Queued.can_be_deleted());
}
#[test]
fn object_key_shape() {
let at = Utc.with_ymd_and_hms(2026, 8, 1, 12, 30, 0).unwrap();
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
assert_eq!(
key,
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
);
}
#[test]
fn object_key_rejects_dangerous_prefixes() {
let at = Utc::now();
assert!(build_object_key("/leading", "backup_x", at).is_err());
assert!(build_object_key("a/../b", "backup_x", at).is_err());
assert!(build_object_key("a\\b", "backup_x", at).is_err());
assert!(build_object_key("a\u{0001}b", "backup_x", at).is_err());
assert!(build_object_key("", "backup_x", at).is_err());
assert!(build_object_key("trailing//", "backup_x", at).is_ok());
}
#[test]
fn trigger_and_status_roundtrip() {
for status in [
BackupStatus::Queued,
BackupStatus::Dumping,
BackupStatus::Uploading,
BackupStatus::Completed,
BackupStatus::Failed,
BackupStatus::Deleting,
] {
assert_eq!(BackupStatus::parse(status.as_str()), Some(status));
}
assert_eq!(
BackupTriggerKind::parse("manual"),
Some(BackupTriggerKind::Manual)
);
assert_eq!(
BackupTriggerKind::parse("scheduled"),
Some(BackupTriggerKind::Scheduled)
);
assert_eq!(BackupTriggerKind::parse("unknown"), None);
assert_eq!(BackupStatus::parse("skipped"), None);
assert_eq!(BackupStatus::parse("deleted"), None);
assert_eq!(BackupStatus::parse("unknown"), None);
}
#[test]
fn config_from_settings_redacts_secret_in_debug() {
let mut settings = BackupSettings {
storage_revision: 3,
endpoint: Some("https://example.com".to_owned()),
region: Some("auto".to_owned()),
bucket: Some("b".to_owned()),
access_key_id: Some("ak".to_owned()),
secret_access_key: Some(SecretString::from("sk-secret")),
prefix: Some("p".to_owned()),
force_path_style: false,
schedule_enabled: false,
cron_expression: None,
schedule_timezone: None,
retention_days: 0,
retention_count: 0,
next_run_at: None,
last_verified_at: None,
updated_at: Utc::now(),
};
assert!(!format!("{:?}", settings).contains("sk-secret"));
let config = BackupStorageConfig::from_settings(&settings).unwrap();
assert!(!format!("{config:?}").contains("sk-secret"));
settings.endpoint = None;
assert!(BackupStorageConfig::from_settings(&settings).is_none());
}
@@ -0,0 +1,3 @@
//! 模型模块测试:镜像 src/model/ 中带逻辑的模块。
mod backup;
@@ -373,7 +373,8 @@ impl ProviderAdmin for FakeProviderAdmin {
pub(super) struct FakeAccountStore {
events: EventLog,
account: AccountRecord,
accounts: Mutex<Vec<AccountRecord>>,
account_after_probe: Mutex<Option<AccountRecord>>,
fail_commit: Mutex<bool>,
audit_requests: Mutex<Vec<String>>,
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
@@ -387,7 +388,8 @@ impl FakeAccountStore {
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
Arc::new(Self {
events,
account,
accounts: Mutex::new(vec![account]),
account_after_probe: Mutex::new(None),
fail_commit: Mutex::new(false),
audit_requests: Mutex::new(Vec::new()),
quota_window_usage: Mutex::new(Vec::new()),
@@ -406,6 +408,17 @@ impl FakeAccountStore {
*self.quota_window_usage.lock().expect("quota window usage") = usage;
}
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
*self.accounts.lock().expect("accounts") = accounts;
}
fn set_account_after_probe(&self, account: AccountRecord) {
*self
.account_after_probe
.lock()
.expect("account after probe") = Some(account);
}
fn record(&self, event: &'static str) {
self.events.lock().expect("store events").push(event);
}
@@ -430,12 +443,14 @@ impl FakeAccountStore {
#[async_trait]
impl AccountStore for FakeAccountStore {
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
self.record("store.list_accounts");
let accounts = self.accounts.lock().expect("accounts").clone();
Ok(AccountPage {
config_revision: revision(1),
items: vec![self.account.clone()],
total: 1,
items: accounts.clone(),
total: accounts.len() as u64,
summary: AccountSummary {
total: 1,
total: accounts.len() as u64,
active: 1,
quota_exhausted: 0,
unavailable: 0,
@@ -445,7 +460,22 @@ impl AccountStore for FakeAccountStore {
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
self.record("store.load_account");
Ok((account_id == self.account.id).then(|| self.account.clone()))
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
let account = self
.account_after_probe
.lock()
.expect("account after probe")
.clone()
.filter(|account| account.id == account_id)
.or_else(|| {
self.accounts
.lock()
.expect("accounts")
.iter()
.find(|account| account.id == account_id)
.cloned()
});
Ok(account)
}
async fn load_account_usage(
@@ -473,11 +503,12 @@ impl AccountStore for FakeAccountStore {
_: CredentialListQuery,
) -> AdminStoreResult<CredentialPage> {
self.record("store.list_credentials");
let accounts = self.accounts.lock().expect("accounts").clone();
Ok(CredentialPage {
config_revision: revision(1),
items: (provider_kind == &self.account.provider_kind)
.then(|| self.account.clone())
items: accounts
.into_iter()
.filter(|account| &account.provider_kind == provider_kind)
.collect(),
next_cursor: None,
})
@@ -489,14 +520,19 @@ impl AccountStore for FakeAccountStore {
account_id: &ProviderAccountId,
) -> AdminStoreResult<Option<CredentialDetails>> {
self.record("store.credential_details");
Ok(
(provider_kind == &self.account.provider_kind
&& account_id.as_str() == self.account.id)
.then(|| CredentialDetails {
config_revision: revision(1),
credential: self.account.clone(),
}),
)
let account = self
.accounts
.lock()
.expect("accounts")
.iter()
.find(|account| {
&account.provider_kind == provider_kind && account.id == account_id.as_str()
})
.cloned();
Ok(account.map(|credential| CredentialDetails {
config_revision: revision(1),
credential,
}))
}
async fn load_credentials_for_export(
@@ -505,21 +541,26 @@ impl AccountStore for FakeAccountStore {
account_ids: &[ProviderAccountId],
) -> AdminStoreResult<Vec<ProviderExportCredentialInput>> {
self.record("store.load_credentials_for_export");
if provider_kind != &self.account.provider_kind
|| account_ids
.iter()
.any(|account_id| account_id.as_str() != self.account.id)
{
let accounts = self.accounts.lock().expect("accounts").clone();
if accounts.iter().any(|account| {
&account.provider_kind != provider_kind
|| !account_ids
.iter()
.any(|account_id| account_id.as_str() == account.id)
}) {
return Err(AdminStoreError::new(
AdminStoreErrorKind::NotFound,
"test account",
"credential not found",
));
}
Ok(vec![ProviderExportCredentialInput {
account: self.account.clone(),
provider_material: document(),
}])
Ok(accounts
.into_iter()
.map(|account| ProviderExportCredentialInput {
account,
provider_material: document(),
})
.collect())
}
async fn commit_credential_import(
@@ -802,6 +843,36 @@ async fn connection_test_should_probe_unavailable_account() {
));
}
#[tokio::test]
async fn connection_test_rate_limited_probe_preserves_active_account_status() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
let probe = Arc::new(FreeModelQuotaProbe {
store: Arc::clone(&store),
});
let services = accounts_service_with_probe(provider, store, probe).await;
let events = services
.accounts()
.test_connection(
ProviderAccountId::new("acct_test").expect("account ID"),
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
)
.await
.expect("connection test stream")
.collect::<Vec<_>>()
.await;
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Active,
..
})
));
}
#[tokio::test]
async fn connection_test_should_preserve_disabled_account_status() {
let provider = FakeProviderAdmin::new("xai", events());
@@ -835,7 +906,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
let provider = FakeProviderAdmin::new("openai", events());
let store = FakeAccountStore::new("openai", events());
let expected = store.account.clone();
let expected = store.accounts.lock().expect("accounts")[0].clone();
accounts_service(provider.clone(), store)
.await
.accounts()
@@ -923,12 +994,76 @@ async fn accounts_list_should_return_complete_directory_semantics() {
}
#[tokio::test]
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() {
let events = events();
let openai = FakeProviderAdmin::new("openai", events.clone());
openai.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
refresh_token_expires_at: None,
windows: vec![ProviderQuotaWindow {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(97.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
limit_reached: false,
local_usage: None,
provider_data: None,
}],
limit_reached: false,
provider_data: None,
});
let failing = FakeProviderAdmin::new("xai", events.clone());
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
let store = FakeAccountStore::new("openai", events.clone());
let mut xai_account = account_record("xai");
xai_account.id = "acct_bad_xai".to_owned();
store.set_accounts(vec![account_record("openai"), xai_account]);
let services = super::AdminHarness::new()
.accounts(store.clone())
.settings(Arc::new(StaticSettingsStore))
.provider(openai)
.provider(failing)
.probe(Arc::new(SuccessfulAccountProbe))
.build()
.await;
let page = services
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
search: None,
status: None,
sort: None,
})
.await
.expect("one failing quota must not fail the directory page");
assert_eq!(page.items.len(), 2);
let by_id = page
.items
.iter()
.map(|item| (item.account.id.as_str(), item))
.collect::<std::collections::HashMap<_, _>>();
let healthy = by_id.get("acct_test").expect("healthy account item");
assert_eq!(healthy.quota.windows.len(), 1);
assert_eq!(healthy.quota.windows[0].key, "primary");
let degraded = by_id.get("acct_bad_xai").expect("degraded account item");
assert!(degraded.quota.windows.is_empty());
assert!(degraded.quota.observed_at.is_none());
assert!(degraded.quota.provider_data.is_none());
assert_eq!(page.summary.total, 2);
}
#[tokio::test]
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Cooldown;
account.availability_reason = Some("usage_limit_exhausted".to_owned());
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
account.availability = AccountAvailability::QuotaExhausted;
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
let store = FakeAccountStore::with_account(account, events());
@@ -952,6 +1087,78 @@ async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_exp
);
}
#[tokio::test]
async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
// Unknown 不可调度,Admin 不得显示为 active。
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Unknown;
let store = FakeAccountStore::with_account(account, events());
let page = accounts_service(provider, store)
.await
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
search: None,
status: None,
sort: None,
})
.await
.expect("unknown account directory");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::Invalid,
);
}
#[tokio::test]
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
// ready + 快照级 quota.limitReached → rate_limited。
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
refresh_token_expires_at: None,
windows: vec![ProviderQuotaWindow {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(100.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
limit_reached: true,
local_usage: None,
provider_data: None,
}],
limit_reached: true,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
let page = accounts_service(provider, store)
.await
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
search: None,
status: None,
sort: None,
})
.await
.expect("rate limited account directory");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::RateLimited,
);
}
#[tokio::test]
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
let provider = FakeProviderAdmin::new("openai", events());
@@ -967,9 +1174,11 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
window_seconds: Some(5 * 60 * 60),
used_percent: Some(97.0),
reset_at: Some(reset_at),
limit_reached: false,
local_usage: None,
provider_data: None,
}],
limit_reached: false,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1077,6 +1286,7 @@ fn empty_quota() -> ProviderQuota {
observed_at: None,
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
provider_data: None,
}
}
@@ -1121,8 +1331,6 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_reason: None,
cooldown_until: None,
availability_observed_at: now,
quota_observed_at: None,
created_at: now,
@@ -1159,8 +1367,6 @@ fn prepared_create_with_id(
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_reason: None,
cooldown_until: None,
availability_observed_at: now,
}
}
@@ -1234,6 +1440,28 @@ impl AccountProbe for FailingAccountProbe {
}
}
struct FreeModelQuotaProbe {
store: Arc<FakeAccountStore>,
}
impl AccountProbe for FreeModelQuotaProbe {
fn probe(
&self,
_: AccountProbeRequest,
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async move {
let mut account = account_record("xai");
account.availability = AccountAvailability::Ready;
self.store.set_account_after_probe(account);
Err(GatewayError::new(
GatewayErrorKind::RateLimited,
"xAI free model quota is exhausted",
)
.into())
})
}
}
fn store_unavailable() -> AdminStoreError {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
@@ -0,0 +1,104 @@
//! BackupService 用例测试:通过 AdminHarness 注入 fake 备份端口构造真实服务。
use std::sync::Arc;
use gateway_admin::model::backup::BackupStatus;
use gateway_admin::ports::backup::BackupStorePorts;
use crate::backup::support::{
FakeAuthStore, FakeBackupRepository, FakeDumpPort, FakeObjectStore, configured_settings,
system_context,
};
use super::AdminHarness;
#[tokio::test]
async fn backup_service_creates_downloads_and_deletes_with_audit() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store = Arc::new(FakeObjectStore::new());
let auth = Arc::new(FakeAuthStore::new());
let services = AdminHarness::new()
.auth(auth.clone())
.backup(BackupStorePorts::new(
repository.clone(),
dump,
object_store,
))
.build()
.await;
// 创建手动备份 → queued + 审计。
let created = services
.backups()
.create_backup(&system_context(), None)
.await
.expect("create backup");
assert_eq!(created.status, BackupStatus::Queued);
assert!(created.object_key.starts_with("codex/"));
assert!(
auth.audit_actions()
.iter()
.any(|action| action == "backup.created")
);
// 下载地址要求 completed 记录。
repository.force_status(&created.id, BackupStatus::Completed);
let download = services
.backups()
.download_url(&system_context(), &created.id)
.await
.expect("download url");
assert!(download.url.starts_with("https://presigned.example.com/"));
assert!(
auth.audit_actions()
.iter()
.any(|action| action == "backup.download_url_created")
);
// 删除 → deleting + 审计。
let deleting = services
.backups()
.delete_backup(&system_context(), &created.id)
.await
.expect("delete backup");
assert_eq!(deleting.status, BackupStatus::Deleting);
assert!(
auth.audit_actions()
.iter()
.any(|action| action == "backup.delete_requested")
);
}
#[tokio::test]
async fn backup_service_rejects_download_of_non_completed_record() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store = Arc::new(FakeObjectStore::new());
let auth = Arc::new(FakeAuthStore::new());
let services = AdminHarness::new()
.auth(auth.clone())
.backup(BackupStorePorts::new(
repository.clone(),
dump,
object_store,
))
.build()
.await;
let created = services
.backups()
.create_backup(&system_context(), None)
.await
.expect("create backup");
// queued 记录不能创建下载地址。
let error = services
.backups()
.download_url(&system_context(), &created.id)
.await
.expect_err("non-completed record must be rejected");
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
}
@@ -1,5 +1,6 @@
mod accounts;
mod auth;
mod backup;
mod client_keys;
mod observability;
mod openai;
@@ -46,6 +47,7 @@ use gateway_admin::{
system::{SystemOperationAccepted, SystemUpdateDetail, SystemUpdateStatus, SystemVersion},
},
ports::{
backup::BackupStorePorts,
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
store::{
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
@@ -75,6 +77,7 @@ pub(super) struct AdminHarness {
client_keys: Arc<dyn ClientKeyStore>,
observability: Arc<dyn ObservabilityStore>,
settings: Arc<dyn SettingsStore>,
backup: BackupStorePorts,
providers: Vec<Arc<dyn ProviderAdmin>>,
probe: Arc<dyn AccountProbe>,
system: Arc<dyn SystemOperations>,
@@ -91,6 +94,7 @@ impl AdminHarness {
client_keys: unavailable.clone(),
observability: unavailable.clone(),
settings: unavailable,
backup: BackupStorePorts::disabled(),
providers: vec![
Arc::new(UnavailableProvider::new("openai")),
Arc::new(UnavailableProvider::new("xai")),
@@ -135,6 +139,11 @@ impl AdminHarness {
self
}
pub(super) fn backup(mut self, backup: BackupStorePorts) -> Self {
self.backup = backup;
self
}
pub(super) fn provider(mut self, provider: Arc<dyn ProviderAdmin>) -> Self {
self.providers
.retain(|registered| registered.provider_kind() != provider.provider_kind());
@@ -165,6 +174,7 @@ impl AdminHarness {
self.client_keys,
self.observability,
self.settings,
self.backup,
),
self.providers,
Arc::new(NoopSnapshot),
@@ -327,9 +327,19 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
let invalid_kind = "x".repeat(65);
store.replace_usage_records(vec![
calculated_total_record("request_invalid_kind", Some(&invalid_kind), now),
calculated_total_record("request_unregistered_kind", Some("anthropic"), now),
calculated_total_record("request_enriched", Some("openai"), now),
total_record(
"request_invalid_kind",
Some(&invalid_kind),
"calculated",
now,
),
total_record(
"request_unregistered_kind",
Some("anthropic"),
"calculated",
now,
),
total_record("request_enriched", Some("openai"), "calculated", now),
]);
let services = observability_services_with_calculated_billing(store).await;
@@ -354,6 +364,30 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
);
}
#[tokio::test]
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
let now = Utc::now();
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
store.replace_usage_records(vec![total_record(
"request_provider_reported",
Some("openai"),
"provider_reported",
now,
)]);
let services = observability_services_with_calculated_billing(store).await;
let page = services
.observability()
.usage_records(usage_query(now))
.await
.expect("usage records");
assert!(matches!(
page.items[0].billing,
Some(UsageBilling::Calculated(_))
));
}
struct FixtureObservabilityStore {
trend: Mutex<Vec<RequestMetricPoint>>,
overview: Mutex<UsageOverview>,
@@ -567,9 +601,10 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
}
}
fn calculated_total_record(
fn total_record(
id: &str,
provider_kind: Option<&str>,
source: &str,
now: DateTime<Utc>,
) -> UsageRecord {
UsageRecord {
@@ -613,11 +648,11 @@ fn calculated_total_record(
image_input_tokens: None,
image_output_tokens: None,
total_tokens: Some(1_000),
cost_source: "calculated".to_owned(),
cost_source: source.to_owned(),
cost_amount: None,
cost_currency: None,
billing: Some(UsageBilling::Total {
source: "calculated".to_owned(),
source: source.to_owned(),
total: CurrencyCost {
currency: "USD".to_owned(),
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
@@ -660,6 +695,7 @@ fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) ->
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
DiagnosticObservation {
key: name.to_owned(),
name: name.to_owned(),
request_count,
success_count: request_count,
@@ -667,6 +703,7 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
attempt_count: request_count,
total_tokens: request_count.saturating_mul(100),
average_latency_ms: Some(100),
latency_p95_ms: Some(200),
cost_coverage: CostCoverage::default(),
costs: Vec::new(),
}
+1
View File
@@ -20,6 +20,7 @@ futures = "0.3.32"
gateway-admin = { path = "../gateway-admin" }
gateway-core = { path = "../gateway-core" }
gateway-protocol = { path = "../gateway-protocol" }
secrecy = "0.10"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
thiserror = "2.0.18"
@@ -16,7 +16,7 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use chrono::{DateTime, FixedOffset, Utc};
use futures::{Stream, StreamExt as _};
use gateway_admin::model::{
AdminError as AdminServiceError, AdminErrorKind, PageSize,
AdminError as AdminServiceError, PageSize,
accounts::{
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
@@ -37,6 +37,7 @@ use gateway_core::{
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use super::presenter::{format_compact_number, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError,
@@ -193,7 +194,6 @@ pub struct AccountView {
pub has_refresh_token: bool,
pub status: String,
pub display_status: String,
pub token_refreshing: bool,
pub availability: String,
pub enabled: bool,
pub access_token_expires_at: Option<String>,
@@ -213,6 +213,7 @@ pub struct AccountView {
#[serde(rename_all = "camelCase")]
pub struct AccountQuotaView {
pub refreshed_at_display: String,
pub limit_reached: bool,
pub windows: Vec<AccountQuotaWindowView>,
}
@@ -226,6 +227,7 @@ pub struct AccountQuotaWindowView {
pub label_display: String,
pub used_percent: Option<f64>,
pub used_percent_display: String,
pub limit_reached: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub local_usage: Option<serde_json::Value>,
pub reset_at_display: String,
@@ -1386,7 +1388,6 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
has_refresh_token: account.has_refresh_token,
status: status.clone(),
display_status: status,
token_refreshing: false,
availability: account.availability.as_str().to_owned(),
enabled: account.enabled,
access_token_expires_at: expires_at,
@@ -1419,6 +1420,7 @@ fn account_quota_view(
(
AccountQuotaView {
refreshed_at_display,
limit_reached: quota.limit_reached,
windows,
},
refresh_token_expires_at,
@@ -1434,6 +1436,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
window_seconds,
used_percent,
reset_at,
limit_reached,
local_usage,
provider_data: _,
} = window;
@@ -1445,6 +1448,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
used_percent,
used_percent_display: used_percent
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
limit_reached,
local_usage: local_usage.as_ref().map(quota_local_usage),
reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)),
}
@@ -1468,7 +1472,7 @@ fn quota_local_usage(usage: &AccountUsage) -> Value {
"imageRequestCount": usage.image_request_count,
"imageRequestFailedCount": usage.image_request_failed_count,
"totalTokens": total_tokens,
"totalTokensDisplay": format_tokens(total_tokens),
"totalTokensDisplay": format_compact_number(total_tokens),
"requestBuckets": usage.request_buckets.iter().map(|bucket| serde_json::json!({
"bucketStart": bucket.bucket_start,
"requestCount": bucket.request_count,
@@ -1601,7 +1605,7 @@ fn account_currency_cost_view(cost: &AccountCost) -> CurrencyCostView {
}
fn display_optional_tokens(value: Option<u64>) -> String {
value.map_or_else(|| "—".to_owned(), format_tokens)
value.map_or_else(|| "—".to_owned(), format_compact_number)
}
fn empty_account_usage() -> AccountUsageView {
@@ -1712,44 +1716,15 @@ fn provider_document_value(document: ProviderDocument) -> Value {
fn account_status_name(status: DomainAccountStatus) -> &'static str {
match status {
DomainAccountStatus::Active => "active",
DomainAccountStatus::Expired => "expired",
DomainAccountStatus::RateLimited => "rate_limited",
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
DomainAccountStatus::Expired => "expired",
DomainAccountStatus::Invalid => "invalid",
DomainAccountStatus::Disabled => "disabled",
DomainAccountStatus::Banned => "banned",
}
}
fn format_number(value: u64) -> String {
let text = value.to_string();
let mut output = String::with_capacity(text.len() + text.len() / 3);
for (index, character) in text.chars().rev().enumerate() {
if index > 0 && index % 3 == 0 {
output.push(',');
}
output.push(character);
}
output.chars().rev().collect()
}
fn format_tokens(value: u64) -> String {
if value < 1_000 {
return format_number(value);
}
for (suffix, threshold) in [
("P", 1_000_000_000_000_000_u64),
("T", 1_000_000_000_000_u64),
("B", 1_000_000_000_u64),
("M", 1_000_000_u64),
("K", 1_000_u64),
] {
if value >= threshold {
let scaled = value as f64 / threshold as f64;
return format!("{scaled:.1}{suffix}").replace(".0", "");
}
}
format_number(value)
}
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
let elapsed = now.signed_duration_since(value);
if elapsed.num_seconds() < 0 {
@@ -1787,16 +1762,5 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
}
fn map_service_error(error: AdminServiceError) -> AdminError {
match error.kind() {
AdminErrorKind::Invalid => AdminError::bad_request(error.to_string()),
AdminErrorKind::Unauthorized => AdminError::admin_session_required(),
AdminErrorKind::NotFound => AdminError::not_found(error.to_string()),
AdminErrorKind::Conflict => AdminError::conflict(error.to_string()),
AdminErrorKind::RateLimited => AdminError::too_many_login_attempts(),
AdminErrorKind::BadGateway => AdminError::bad_gateway(error.to_string()),
AdminErrorKind::Unavailable => {
AdminError::service_unavailable("Account directory unavailable")
}
AdminErrorKind::Internal => AdminError::internal(error.to_string()),
}
super::wire::map_admin_service_error(error, "Account directory unavailable")
}
@@ -0,0 +1,459 @@
//! 备份设置、计划、手动创建、记录、下载与删除路由。
//!
//! 路径统一位于 `/api/admin/settings/backups/*`,内部由独立 `BackupService` 承担业务。
use axum::{
Json, Router,
extract::{Query, State},
http::StatusCode,
response::IntoResponse,
routing::{get, post},
};
use chrono::{DateTime, Utc};
use gateway_admin::model::backup::{
BackupRecord, BackupSettings, BackupStatus, BackupTriggerKind, ConnectionTestResult,
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
};
use secrecy::{ExposeSecret as _, SecretString};
use serde::{Deserialize, Serialize};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminPageData, AdminResponse, AdminSessionState,
PageMeta, wire::map_admin_service_error,
};
/// 备份设置视图;`secretAccessKey` 返回已保存的明文凭据,由前端掩码显示。
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BackupSettingsView {
pub storage_revision: u64,
pub endpoint: Option<String>,
pub region: Option<String>,
pub bucket: Option<String>,
pub access_key_id: Option<String>,
pub secret_access_key: Option<String>,
pub prefix: Option<String>,
pub force_path_style: bool,
pub verified: bool,
pub schedule_enabled: bool,
pub cron_expression: Option<String>,
pub schedule_timezone: Option<String>,
pub retention_days: u32,
pub retention_count: u32,
pub next_run_at: Option<DateTime<Utc>>,
pub last_verified_at: Option<DateTime<Utc>>,
pub updated_at: DateTime<Utc>,
}
impl From<BackupSettings> for BackupSettingsView {
fn from(settings: BackupSettings) -> Self {
let verified = settings.storage_verified();
Self {
storage_revision: settings.storage_revision,
endpoint: settings.endpoint,
region: settings.region,
bucket: settings.bucket,
access_key_id: settings.access_key_id,
secret_access_key: settings
.secret_access_key
.map(|secret| secret.expose_secret().to_owned()),
prefix: settings.prefix,
force_path_style: settings.force_path_style,
verified,
schedule_enabled: settings.schedule_enabled,
cron_expression: settings.cron_expression,
schedule_timezone: settings.schedule_timezone,
retention_days: settings.retention_days,
retention_count: settings.retention_count,
next_run_at: settings.next_run_at,
last_verified_at: settings.last_verified_at,
updated_at: settings.updated_at,
}
}
}
/// 更新 S3 存储配置请求;`secretAccessKey` 缺省表示保留旧值。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UpdateBackupStorageRequest {
pub endpoint: String,
pub region: String,
pub bucket: String,
pub access_key_id: String,
#[serde(default)]
pub secret_access_key: Option<String>,
pub prefix: String,
pub force_path_style: bool,
}
/// 更新调度配置请求。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UpdateBackupScheduleRequest {
pub schedule_enabled: bool,
pub cron_expression: String,
pub schedule_timezone: String,
pub retention_days: u32,
pub retention_count: u32,
}
/// 备份记录 wire 视图。
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BackupRecordView {
pub id: String,
pub trigger_kind: String,
pub status: String,
pub scheduled_at: Option<DateTime<Utc>>,
pub object_key: String,
pub size_bytes: Option<u64>,
pub sha256: Option<String>,
pub attempt_count: u32,
pub error_code: Option<String>,
pub error_message: Option<String>,
pub started_at: Option<DateTime<Utc>>,
pub completed_at: Option<DateTime<Utc>>,
pub expires_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
impl From<BackupRecord> for BackupRecordView {
fn from(record: BackupRecord) -> Self {
Self {
id: record.id,
trigger_kind: record.trigger_kind.to_string(),
status: record.status.to_string(),
scheduled_at: record.scheduled_at,
object_key: record.object_key,
size_bytes: record.size_bytes,
sha256: record.sha256,
attempt_count: record.attempt_count,
error_code: record.error_code,
error_message: record.error_message,
started_at: record.started_at,
completed_at: record.completed_at,
expires_at: record.expires_at,
created_at: record.created_at,
updated_at: record.updated_at,
}
}
}
/// 记录列表查询参数。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackupRecordsQuery {
pub page: Option<u32>,
pub page_size: Option<u16>,
pub status: Option<String>,
pub trigger: Option<String>,
}
/// 下载地址请求。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BackupIdRequest {
pub backup_id: String,
}
/// 下载地址响应。
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DownloadUrlView {
pub url: String,
pub file_name: String,
pub expires_in_seconds: u64,
}
impl From<gateway_admin::model::backup::DownloadUrlResult> for DownloadUrlView {
fn from(result: gateway_admin::model::backup::DownloadUrlResult) -> Self {
Self {
url: result.url,
file_name: result.file_name,
expires_in_seconds: result.expires_in.as_secs(),
}
}
}
/// 连接测试响应。
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ConnectionTestView {
pub ok: bool,
pub stage: String,
pub code: Option<String>,
pub message: String,
}
impl From<ConnectionTestResult> for ConnectionTestView {
fn from(result: ConnectionTestResult) -> Self {
Self {
ok: result.ok,
stage: result.stage.to_owned(),
code: result.code.map(str::to_owned),
message: result.message,
}
}
}
/// 构造固定 GET/POST 备份路由。
pub fn router<S>() -> Router<S>
where
S: AdminSessionState + Clone + Send + Sync + 'static,
{
Router::new()
.route("/api/admin/settings/backups", get(backup_settings::<S>))
.route(
"/api/admin/settings/backups/storage/update",
post(update_backup_storage::<S>),
)
.route(
"/api/admin/settings/backups/storage/test",
post(test_backup_storage::<S>),
)
.route(
"/api/admin/settings/backups/schedule/update",
post(update_backup_schedule::<S>),
)
.route(
"/api/admin/settings/backups/records",
get(backup_records::<S>),
)
.route(
"/api/admin/settings/backups/create",
post(create_backup::<S>),
)
.route(
"/api/admin/settings/backups/download-url",
post(download_backup_url::<S>),
)
.route(
"/api/admin/settings/backups/delete",
post(delete_backup::<S>),
)
}
async fn backup_settings<S>(
_auth: AdminAuth,
State(state): State<S>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let settings = state
.admin_services()
.backups()
.load_settings()
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BackupSettingsView::from(settings)),
))
}
async fn update_backup_storage<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<UpdateBackupStorageRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let command = UpdateBackupStorageCommand {
endpoint: request.endpoint,
region: request.region,
bucket: request.bucket,
access_key_id: request.access_key_id,
secret_access_key: request.secret_access_key.map(SecretString::from),
prefix: request.prefix,
force_path_style: request.force_path_style,
};
let settings = state
.admin_services()
.backups()
.update_storage(&auth.context().mutation_context(), command)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BackupSettingsView::from(settings)),
))
}
async fn test_backup_storage<S>(
auth: AdminAuth,
State(state): State<S>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let result = state
.admin_services()
.backups()
.test_connection(&auth.context().mutation_context())
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(ConnectionTestView::from(result)),
))
}
async fn update_backup_schedule<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<UpdateBackupScheduleRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let command = UpdateBackupScheduleCommand {
schedule_enabled: request.schedule_enabled,
cron_expression: request.cron_expression,
schedule_timezone: request.schedule_timezone,
retention_days: request.retention_days,
retention_count: request.retention_count,
};
let settings = state
.admin_services()
.backups()
.update_schedule(&auth.context().mutation_context(), command)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BackupSettingsView::from(settings)),
))
}
async fn backup_records<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<BackupRecordsQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let page_size = gateway_admin::model::PageSize::new(query.page_size.unwrap_or(20))
.map_err(|_| AdminError::bad_request("Invalid pageSize"))?;
let page = query.page.unwrap_or(1);
if page == 0 {
return Err(AdminError::bad_request("Invalid page"));
}
let status = match query.status.as_deref() {
None => None,
Some(value) => BackupStatus::parse(value)
.map(Some)
.ok_or_else(|| AdminError::bad_request("Invalid status"))?,
};
let trigger = match query.trigger.as_deref() {
None => None,
Some(value) => BackupTriggerKind::parse(value)
.map(Some)
.ok_or_else(|| AdminError::bad_request("Invalid trigger"))?,
};
let domain_query = gateway_admin::model::backup::BackupRecordListQuery {
page,
page_size,
status,
trigger,
};
let page_result = state
.admin_services()
.backups()
.list_records(domain_query)
.await
.map_err(map_service_error)?;
let meta = PageMeta::new(
page_result.page,
u32::from(page_result.page_size.get()),
page_result.total,
page_result.total_pages(),
);
let items: Vec<BackupRecordView> = page_result
.items
.into_iter()
.map(BackupRecordView::from)
.collect();
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AdminPageData::new(items, meta)),
))
}
async fn create_backup<S>(
auth: AdminAuth,
State(state): State<S>,
body: Option<Json<CreateBackupRequest>>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let expires_at = body
.and_then(|body| body.0.expires_in_days)
.filter(|&days| days > 0)
.map(|days| Utc::now() + chrono::Duration::days(i64::from(days)));
let record = state
.admin_services()
.backups()
.create_backup(&auth.context().mutation_context(), expires_at)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::ACCEPTED,
AdminEnvelope::ok(BackupRecordView::from(record)),
))
}
/// 创建备份请求;`expiresInDays` 为 0 或缺省表示不设置过期时间。
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CreateBackupRequest {
pub expires_in_days: Option<u32>,
}
async fn download_backup_url<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<BackupIdRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let result = state
.admin_services()
.backups()
.download_url(&auth.context().mutation_context(), &request.backup_id)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(DownloadUrlView::from(result)),
))
}
async fn delete_backup<S>(
auth: AdminAuth,
State(state): State<S>,
Json(request): Json<BackupIdRequest>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let record = state
.admin_services()
.backups()
.delete_backup(&auth.context().mutation_context(), &request.backup_id)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BackupRecordView::from(record)),
))
}
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
map_admin_service_error(error, "Backup service unavailable")
}
@@ -9,8 +9,10 @@ use axum::{
pub mod accounts;
pub mod auth;
pub mod backups;
pub mod client_keys;
pub mod observability;
pub mod presenter;
pub mod settings;
pub mod system;
pub mod wire;
@@ -29,6 +31,7 @@ where
Router::new()
.merge(accounts::router::<S>())
.merge(auth::router::<S>())
.merge(backups::router::<S>())
.merge(client_keys::router::<S>())
.merge(observability::router::<S>())
.merge(settings::router::<S>())
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,32 @@
//! Dashboard、用量与诊断查询的 wire 映射和固定路由。
use std::{collections::BTreeMap, num::NonZeroU32};
use axum::{
Router,
extract::{Query, State},
http::StatusCode,
response::IntoResponse,
routing::get,
};
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use chrono::{DateTime, Duration, Utc};
use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::presenter::{format_compact_number, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError, wire::map_admin_service_error,
};
mod presenter;
mod query;
mod routes;
mod wire;
pub(crate) use presenter::*;
pub use query::*;
pub use routes::*;
pub use wire::*;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,463 @@
//! 查询 wire 类型、校验与领域命令映射。
use super::*;
use base64::Engine as _;
/// 观测列表默认页大小。
pub const DEFAULT_PAGE_SIZE: u16 = 50;
/// 观测列表允许的最大页大小。
pub const MAX_PAGE_SIZE: u16 = 100;
/// 游标编码在 HTTP 层解码前允许的最大长度。
pub const MAX_CURSOR_BYTES: usize = 1024;
/// Dashboard 查询参数。
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct DashboardQuery {
pub kind: Option<String>,
pub start_time: Option<String>,
pub end_time: Option<String>,
}
impl DashboardQuery {
/// 解析 dashboard 趋势类型。
pub fn trend_kind(&self) -> Result<TrendKind, WireValidationError> {
TrendKind::parse(self.kind.as_deref())
}
}
/// 逻辑请求列表、汇总和洞察查询参数。
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UsageQuery {
pub page: Option<u32>,
pub page_size: Option<u16>,
pub cursor: Option<String>,
pub kind: Option<String>,
pub outcome: Option<String>,
pub client_api_key_id: Option<String>,
pub provider: Option<String>,
pub request_id: Option<String>,
pub account_id: Option<String>,
pub route: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub transport: Option<String>,
pub attempt_index: Option<i64>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub search: Option<String>,
pub start_time: Option<String>,
pub end_time: Option<String>,
}
impl UsageQuery {
/// 校验分页字段,不改变仓储层的分页 owner。
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
let page = self.page.unwrap_or(1);
if page == 0 {
return Err(WireValidationError::new("page"));
}
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
if page_size == 0 || page_size > MAX_PAGE_SIZE {
return Err(WireValidationError::new("pageSize"));
}
Ok((page, page_size))
}
/// 校验游标的 wire 边界;编码和排序语义由应用层负责。
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
validate_cursor(self.cursor.as_deref())
}
}
/// 详情查询参数。
#[derive(Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct DetailQuery {
pub id: String,
}
impl DetailQuery {
/// 校验详情 ID,错误不回显输入值。
pub fn validate(&self) -> Result<(), WireValidationError> {
require_text(&self.id, "id")
}
}
/// 诊断聚合查询参数。
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct DiagnosticsQuery {
pub dimension: Option<String>,
pub start_time: Option<String>,
pub end_time: Option<String>,
pub provider: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub search: Option<String>,
}
impl DiagnosticsQuery {
/// 解析诊断维度。
pub fn dimension(&self) -> Result<DiagnosticDimension, WireValidationError> {
DiagnosticDimension::parse(self.dimension.as_deref())
}
}
/// 运维错误查询参数。
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct OpsQuery {
pub page: Option<u32>,
pub page_size: Option<u16>,
pub cursor: Option<String>,
pub kind: Option<String>,
pub client_api_key_id: Option<String>,
pub provider: Option<String>,
pub request_id: Option<String>,
pub account_id: Option<String>,
pub route: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub client_status_code: Option<i64>,
pub upstream_status_code: Option<i64>,
pub transport: Option<String>,
pub attempt_index: Option<i64>,
pub failure_class: Option<String>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub search: Option<String>,
pub start_time: Option<String>,
pub end_time: Option<String>,
}
impl OpsQuery {
/// 校验分页字段。
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
let page = self.page.unwrap_or(1);
if page == 0 {
return Err(WireValidationError::new("page"));
}
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
if page_size == 0 || page_size > MAX_PAGE_SIZE {
return Err(WireValidationError::new("pageSize"));
}
Ok((page, page_size))
}
/// 校验游标的 wire 边界。
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
validate_cursor(self.cursor.as_deref())
}
}
/// Dashboard 趋势类型。
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub enum TrendKind {
Usage,
Latency,
Errors,
}
impl TrendKind {
/// 从 query 值解析趋势类型。
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
match trimmed(value) {
None | Some("usage") => Ok(Self::Usage),
Some("latency") => Ok(Self::Latency),
Some("errors") => Ok(Self::Errors),
Some(_) => Err(WireValidationError::new("kind")),
}
}
}
/// 诊断聚合维度。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DiagnosticDimension {
Model,
Account,
ApiKey,
Provider,
Transport,
Failure,
Status,
}
impl DiagnosticDimension {
/// 从 query 值解析诊断维度。
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
match trimmed(value) {
None | Some("model") => Ok(Self::Model),
Some("account") => Ok(Self::Account),
Some("apiKey" | "api_key") => Ok(Self::ApiKey),
Some("provider") => Ok(Self::Provider),
Some("transport") => Ok(Self::Transport),
Some("failureClass" | "failure_class") => Ok(Self::Failure),
Some("status") => Ok(Self::Status),
Some(_) => Err(WireValidationError::new("dimension")),
}
}
/// 返回终态响应中的稳定维度名称。
#[must_use]
pub const fn display_name(self) -> &'static str {
match self {
Self::Model => "model",
Self::Account => "account",
Self::ApiKey => "apiKey",
Self::Provider => "provider",
Self::Transport => "transport",
Self::Failure => "failureClass",
Self::Status => "status",
}
}
}
/// 解析 RFC3339 时间;错误不回显原始值。
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
let Some(value) = trimmed(value) else {
return Ok(None);
};
DateTime::parse_from_rfc3339(value)
.map(|value| Some(value.with_timezone(&Utc)))
.map_err(|_| WireValidationError::new("timeRange"))
}
/// 解析 HTTP 状态码。
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
value
.map(|value| {
u16::try_from(value)
.ok()
.filter(|value| (100..=599).contains(value))
.ok_or_else(|| WireValidationError::new("statusCode"))
})
.transpose()
}
/// 解析尝试序号。
pub fn parse_attempt_index(value: Option<i64>) -> Result<Option<u32>, WireValidationError> {
value
.map(|value| {
u32::try_from(value)
.ok()
.filter(|value| *value > 0 && *value <= i32::MAX as u32)
.ok_or_else(|| WireValidationError::new("attemptIndex"))
})
.transpose()
}
pub(crate) fn require_text(value: &str, field: &'static str) -> Result<(), WireValidationError> {
if value.trim().is_empty() {
Err(WireValidationError::new(field))
} else {
Ok(())
}
}
pub(crate) fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> {
if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) {
return Err(WireValidationError::new("cursor"));
}
Ok(())
}
pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
pub(crate) fn domain_trend_kind(kind: TrendKind) -> domain::TrendKind {
match kind {
TrendKind::Usage => domain::TrendKind::Usage,
TrendKind::Latency => domain::TrendKind::Latency,
TrendKind::Errors => domain::TrendKind::Errors,
}
}
pub(crate) fn domain_diagnostic_dimension(
dimension: DiagnosticDimension,
) -> domain::DiagnosticDimension {
match dimension {
DiagnosticDimension::Model => domain::DiagnosticDimension::Model,
DiagnosticDimension::Account => domain::DiagnosticDimension::Account,
DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey,
DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider,
DiagnosticDimension::Transport => domain::DiagnosticDimension::Transport,
DiagnosticDimension::Failure => domain::DiagnosticDimension::Failure,
DiagnosticDimension::Status => domain::DiagnosticDimension::Status,
}
}
pub(crate) fn request_outcome(
value: Option<String>,
) -> Result<Option<domain::RequestOutcome>, WireValidationError> {
non_empty(value)
.map(|value| {
domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome"))
})
.transpose()
}
pub(crate) fn usage_range(
start: Option<&str>,
end: Option<&str>,
) -> Result<domain::TimeRange, WireValidationError> {
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
let start = parse_datetime(start)?.unwrap_or(end - Duration::days(7));
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
}
pub(crate) fn dashboard_range(
start: Option<&str>,
end: Option<&str>,
) -> Result<domain::TimeRange, WireValidationError> {
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
let start =
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
}
pub(crate) fn dashboard_today_range(
start: Option<&str>,
end: Option<&str>,
) -> Result<domain::TimeRange, WireValidationError> {
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
let start = parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end));
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
}
pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, WireValidationError> {
let outcome = query.outcome.clone().or_else(|| {
non_empty(query.kind.clone()).filter(|value| {
matches!(
value.as_str(),
"running" | "succeeded" | "failed" | "cancelled" | "incomplete"
)
})
});
Ok(domain::UsageFilter {
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
request_id: non_empty(query.request_id.clone()),
provider_account_ref: non_empty(query.account_id.clone()),
operation: non_empty(query.route.clone()),
provider_kind: non_empty(query.provider.clone()),
model: non_empty(query.model.clone()),
outcome: request_outcome(outcome)?,
status_code: parse_status(query.status_code)?,
transport: non_empty(query.transport.clone()),
attempt_index: parse_attempt_index(query.attempt_index)?,
response_id: non_empty(query.response_id.clone()),
upstream_request_id: non_empty(query.upstream_request_id.clone()),
search: non_empty(query.search.clone()),
})
}
pub(crate) fn usage_command(
query: &UsageQuery,
) -> Result<(domain::UsageQuery, u32, u16), WireValidationError> {
let (page, page_size) = query.validate_page()?;
query.validate_cursor()?;
let page_number = NonZeroU32::new(page)
.map(domain::PageNumber::new)
.ok_or_else(|| WireValidationError::new("page"))?;
let page_size_value =
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
Ok((
domain::UsageQuery {
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
filter: usage_filter(query)?,
cursor: decode_observability_cursor(query.cursor.as_deref())?,
page: page_number,
page_size: page_size_value,
},
page,
page_size,
))
}
pub(crate) fn ops_command(
query: &OpsQuery,
) -> Result<(domain::OpsErrorQuery, u32, u16), WireValidationError> {
let (page, page_size) = query.validate_page()?;
query.validate_cursor()?;
let page_number = NonZeroU32::new(page)
.map(domain::PageNumber::new)
.ok_or_else(|| WireValidationError::new("page"))?;
let page_size_value =
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
let status_code = parse_status(
query
.upstream_status_code
.or(query.client_status_code)
.or(query.status_code),
)?;
Ok((
domain::OpsErrorQuery {
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
filter: domain::OpsErrorFilter {
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
request_id: non_empty(query.request_id.clone()),
provider_kind: non_empty(query.provider.clone()),
provider_account_ref: non_empty(query.account_id.clone()),
operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())),
model: non_empty(query.model.clone()),
transport: non_empty(query.transport.clone()),
attempt_index: parse_attempt_index(query.attempt_index)?,
response_id: non_empty(query.response_id.clone()),
upstream_request_id: non_empty(query.upstream_request_id.clone()),
failure_kind: non_empty(query.failure_class.clone()),
status_code,
search: non_empty(query.search.clone()),
},
cursor: decode_observability_cursor(query.cursor.as_deref())?,
page: page_number,
page_size: page_size_value,
},
page,
page_size,
))
}
pub(crate) fn decode_observability_cursor(
value: Option<&str>,
) -> Result<Option<domain::ObservabilityCursor>, WireValidationError> {
value
.map(|encoded| {
if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES {
return Err(WireValidationError::new("cursor"));
}
let bytes = URL_SAFE_NO_PAD
.decode(encoded)
.map_err(|_| WireValidationError::new("cursor"))?;
let wire: CursorWire =
serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?;
if wire.stable_id.trim().is_empty() {
return Err(WireValidationError::new("cursor"));
}
Ok(domain::ObservabilityCursor {
observed_at: wire.observed_at,
stable_id: wire.stable_id,
})
})
.transpose()
}
pub(crate) fn encode_observability_cursor(
cursor: &domain::ObservabilityCursor,
) -> Result<String, WireValidationError> {
let bytes = serde_json::to_vec(&CursorWire {
observed_at: cursor.observed_at,
stable_id: cursor.stable_id.clone(),
})
.map_err(|_| WireValidationError::new("cursor"))?;
Ok(URL_SAFE_NO_PAD.encode(bytes))
}
pub(crate) fn non_empty(value: Option<String>) -> Option<String> {
value
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty())
}
@@ -0,0 +1,213 @@
//! 固定 `/api/admin` 观测路由与 handler。
use super::*;
pub fn router<S>() -> Router<S>
where
S: AdminSessionState + Clone + Send + Sync + 'static,
{
Router::new()
.route("/api/admin/dashboard/summary", get(dashboard_summary::<S>))
.route("/api/admin/dashboard/trend", get(dashboard_trend::<S>))
.route("/api/admin/usage/records", get(usage_records::<S>))
.route(
"/api/admin/usage/records/detail",
get(usage_record_detail::<S>),
)
.route(
"/api/admin/usage/records/summary",
get(usage_records_summary::<S>),
)
.route(
"/api/admin/usage/insights/overview",
get(usage_insights_overview::<S>),
)
.route(
"/api/admin/usage/insights/diagnostics",
get(usage_insights_diagnostics::<S>),
)
.route("/api/admin/operations/errors", get(ops_errors::<S>))
}
pub(crate) async fn dashboard_summary<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<DashboardQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let kind = query.trend_kind().map_err(map_wire_error)?;
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.dashboard_summary(range, domain_trend_kind(kind))
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(dashboard_view(result, kind)),
))
}
pub(crate) async fn dashboard_trend<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<DashboardQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let kind = query.trend_kind().map_err(map_wire_error)?;
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.dashboard_trend(range, domain_trend_kind(kind))
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(trend_view(result, kind)),
))
}
pub(crate) async fn usage_records<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<UsageQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let (command, page, page_size) = usage_command(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.usage_records(command)
.await
.map_err(map_service_error)?;
let data = usage_page_view(result, page, page_size)
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
}
pub(crate) async fn usage_record_detail<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<DetailQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
query.validate().map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.usage_record_detail(query.id.trim())
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(usage_detail_view(result)),
))
}
pub(crate) async fn usage_records_summary<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<UsageQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let filter = usage_filter(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.usage_summary(range, filter)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(usage_summary_view(result)),
))
}
pub(crate) async fn usage_insights_overview<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<UsageQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let filter = usage_filter(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.usage_insights(range, filter)
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(usage_insights_view(result)),
))
}
pub(crate) async fn usage_insights_diagnostics<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<DiagnosticsQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let dimension = query.dimension().map_err(map_wire_error)?;
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
.map_err(map_wire_error)?;
let filter = domain::UsageFilter {
provider_kind: non_empty(query.provider),
model: non_empty(query.model),
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
search: non_empty(query.search),
..domain::UsageFilter::default()
};
let result = state
.admin_services()
.observability()
.diagnostics(range, filter, domain_diagnostic_dimension(dimension))
.await
.map_err(map_service_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(diagnostics_view(result, dimension)),
))
}
pub(crate) async fn ops_errors<S>(
_auth: AdminAuth,
State(state): State<S>,
Query(query): Query<OpsQuery>,
) -> Result<impl IntoResponse, AdminError>
where
S: AdminSessionState + Send + Sync,
{
let (command, page, page_size) = ops_command(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
.ops_errors(command)
.await
.map_err(map_service_error)?;
let data = ops_page_view(result, page, page_size)
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
}
@@ -0,0 +1,712 @@
//! 响应 DTO 与固定 wire 形状。
use super::*;
/// 观测列表游标的稳定 wire 形状。
#[derive(Clone, Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CursorWire {
pub observed_at: DateTime<Utc>,
pub stable_id: String,
}
/// 观测列表响应数据。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PageData<T> {
pub items: Vec<T>,
pub page: PageMeta,
pub next_cursor: Option<String>,
}
/// Token 详情展示。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TokenDetailsView {
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub image_input_tokens: Option<u64>,
pub image_output_tokens: Option<u64>,
pub total_tokens: Option<u64>,
pub input_tokens_display: String,
pub output_tokens_display: String,
pub cached_tokens_display: String,
pub cache_write_tokens_display: String,
pub reasoning_tokens_display: String,
pub image_input_tokens_display: String,
pub image_output_tokens_display: String,
pub total_tokens_display: String,
}
/// 按货币展示的成本。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CostView {
pub currency: String,
pub estimated_amount: String,
}
/// 成本覆盖状态计数。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CostCoverageView {
pub known: u64,
pub partial: u64,
pub unknown: u64,
pub not_billable: u64,
}
/// Provider 受控价格规则生成的单次请求费用明细展示。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BillingView {
pub input_amount_display: String,
pub output_amount_display: String,
pub cache_read_amount_display: String,
pub cache_write_amount_display: String,
pub standard_amount_display: String,
pub total_amount_display: String,
pub input_price_display: String,
pub output_price_display: String,
pub cache_read_price_display: String,
pub cache_write_price_display: String,
pub service_tier_display: String,
pub multiplier_display: String,
}
/// 单条逻辑请求展示。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageRecordView {
pub id: String,
pub request_id: String,
pub client_api_key_id: Option<String>,
pub kind: String,
pub provider: Option<String>,
pub authentication_kind: Option<String>,
pub account_id: Option<String>,
pub account_email: Option<String>,
pub account_name: Option<String>,
pub route: String,
pub model: String,
pub requested_model: Option<String>,
pub upstream_model: Option<String>,
pub service_tier: Option<String>,
pub status_code: Option<i64>,
pub transport: Option<String>,
pub protocol: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub http_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub client_status_code: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub upstream_status_code: Option<i64>,
pub websocket_pool: Option<WebSocketPoolMetadataView>,
pub image_generation_requested: bool,
pub image_generation_succeeded: Option<bool>,
pub latency_details: UsageLatencyDetailsView,
pub attempt_index: Option<i64>,
pub attempt_count: u64,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub first_token_ms: Option<u64>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub image_input_tokens: Option<u64>,
pub image_output_tokens: Option<u64>,
pub message: String,
/// Provider 安全观测;Core 字段由顶层字段提供,不再复制进 metadata。
pub metadata: BTreeMap<String, Value>,
pub created_at: DateTime<Utc>,
pub created_at_display: String,
pub client_ip: Option<String>,
pub user_agent: Option<String>,
pub reasoning_effort: Option<String>,
pub reasoning_preset: Option<String>,
pub compact: Option<bool>,
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub token_details: TokenDetailsView,
pub billing: Option<BillingView>,
pub costs: Vec<CostView>,
pub cost_coverage: CostCoverageView,
pub first_token_latency_ms: Option<u64>,
pub first_token_latency_ms_display: String,
pub latency_ms_display: String,
pub logical_outcome: String,
}
/// WebSocket 池决策的稳定形状;与 Provider 选择逻辑无关。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct WebSocketPoolMetadataView {
pub kind: String,
}
/// 逻辑请求在上游和输出阶段测得的时延事实。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageLatencyDetailsView {
#[serde(skip_serializing_if = "Option::is_none")]
pub transport_decision_wait_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ws_connect_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub upstream_headers_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub first_event_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub first_reasoning_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub first_text_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub first_token_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub openai_processing_ms: Option<u64>,
}
/// 单次上游尝试展示。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageAttemptView {
pub id: String,
pub attempt_index: u32,
pub trigger: String,
pub provider: String,
pub model: String,
pub transport: String,
pub send_state: String,
pub outcome: String,
pub downstream_committed: bool,
pub status_code: Option<u16>,
pub provider_error_code: Option<String>,
pub failure_class: Option<String>,
pub cost_estimate_status: String,
pub estimated_cost_amount: Option<String>,
pub estimated_cost_currency: Option<String>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub total_tokens: Option<u64>,
pub first_token_ms: Option<u64>,
pub latency_ms: Option<u64>,
pub credential_name: Option<String>,
pub account_id: Option<String>,
pub account_name: Option<String>,
pub account_email: Option<String>,
pub authentication_kind: Option<String>,
pub started_at: DateTime<Utc>,
pub completed_at: Option<DateTime<Utc>>,
}
/// 逻辑请求详情与其尝试列表。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageRecordDetailView {
#[serde(flatten)]
pub request: UsageRecordView,
pub attempts: Vec<UsageAttemptView>,
/// 尝试列表是否完整;best-effort 下恒为 false。
pub attempts_complete: bool,
}
/// Dashboard 趋势数据。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TrendData {
pub kind: TrendKind,
pub points: Vec<TrendPointView>,
pub summary: Vec<TrendSummaryView>,
}
/// Dashboard 单个趋势桶。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TrendPointView {
pub time: String,
pub bucket: DateTime<Utc>,
pub label: String,
pub requests: String,
pub requests_value: u64,
pub input_tokens: String,
pub input_tokens_value: u64,
pub output_tokens: String,
pub output_tokens_value: u64,
pub cached_tokens: String,
pub cached_tokens_value: u64,
pub cache_hit_rate_value: f64,
pub tokens_value: u64,
pub errors: String,
pub errors_value: u64,
pub latency: String,
pub latency_value: Option<u64>,
pub max_latency: String,
pub max_latency_value: Option<u64>,
pub min_latency: String,
pub min_latency_value: Option<u64>,
pub success_rate: String,
pub success_rate_value: Option<f64>,
}
/// Dashboard 趋势摘要。
#[derive(Debug, Serialize)]
pub struct TrendSummaryView {
pub label: String,
pub value: String,
pub ratio: Option<String>,
}
/// Dashboard 卡片集合。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardCardsView {
pub credentials: DashboardCredentialsCardView,
pub traffic: DashboardTrafficCardView,
pub tokens: DashboardTokensCardView,
pub cache: DashboardCacheCardView,
}
/// Dashboard 上游凭据卡片。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardCredentialsCardView {
pub total: String,
pub total_value: u64,
pub available: String,
pub available_value: u64,
pub unavailable: String,
pub unavailable_value: u64,
}
/// Dashboard 流量卡片。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardTrafficCardView {
pub today_requests: String,
pub today_requests_value: u64,
pub yesterday_requests_value: u64,
pub total_requests: String,
}
/// Dashboard token 卡片。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardTokensCardView {
pub today_tokens: String,
pub today_tokens_value: u64,
pub yesterday_tokens_value: u64,
pub total_tokens: String,
pub total_billing_amount_usd: String,
}
/// Dashboard 缓存卡片。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardCacheCardView {
pub today_hit_rate: String,
pub today_hit_rate_value: Option<f64>,
pub yesterday_hit_rate_value: Option<f64>,
pub total_hit_rate: String,
pub total_cached_tokens: String,
pub average_first_token_latency_ms: String,
}
/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardAccountUsageView {
pub id: String,
pub provider: String,
pub authentication_kind: String,
pub email: String,
pub plan_type: Option<String>,
pub tokens: String,
pub request_count: u64,
pub request_buckets: Vec<DashboardAccountRequestBucketView>,
pub quota_used_percent: Option<f64>,
pub last_used: String,
}
/// Dashboard 账号单小时请求数。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardAccountRequestBucketView {
pub bucket_start: DateTime<Utc>,
pub request_count: u64,
}
/// Provider 账号池的持久事实汇总。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardPoolSummaryView {
pub total: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub disabled: u64,
pub banned: u64,
}
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardCapacityInfoView {
pub max_concurrent_per_account: u64,
pub total_slots: u64,
pub used_slots: Option<u64>,
pub available_slots: Option<u64>,
}
/// 逻辑请求指标展示。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct RequestMetricsView {
pub request_count: u64,
pub success_count: u64,
pub failure_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub caller_error_count: u64,
pub input_tokens: u64,
pub output_tokens: u64,
pub cached_tokens: u64,
pub cache_write_tokens: u64,
pub reasoning_tokens: u64,
pub total_tokens: u64,
}
/// 上游尝试指标展示。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AttemptMetricsView {
pub attempt_count: u64,
pub success_count: u64,
pub failure_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub rate_limited_count: u64,
pub auth_failure_count: u64,
pub provider5xx_count: u64,
pub cost_coverage: CostCoverageView,
pub costs: Vec<CostView>,
}
/// 健康时间线单点。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HealthTimelinePointView {
pub time: String,
pub status: String,
pub reliability_display: String,
pub success_requests: u64,
pub failed_requests: u64,
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
}
/// Dashboard 健康时间线。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HealthTimelineView {
pub title: String,
pub description: String,
pub reliability_display: String,
pub status: String,
pub success_requests: u64,
pub failed_requests: u64,
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub points: Vec<HealthTimelinePointView>,
}
/// Dashboard 展示的实际 Provider 上游请求身份。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardWireProfileView {
pub provider: String,
pub product: String,
pub version: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub build: Option<String>,
pub target: DashboardWireTargetView,
pub user_agent: String,
pub attributes: Vec<DashboardWireAttributeView>,
#[serde(skip_serializing_if = "Option::is_none")]
pub verified_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub release: Option<DashboardDesktopReleaseView>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardWireAttributeView {
pub label: String,
pub value: String,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardWireTargetView {
pub os_type: String,
pub os_version: String,
pub arch: String,
pub terminal: String,
}
/// 发布检查与启动画像分离;未检查时使用明确的 `unchecked` 状态。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardDesktopReleaseView {
pub status: DashboardDesktopReleaseStatusView,
#[serde(skip_serializing_if = "Option::is_none")]
pub checked_at: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub latest_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub latest_build: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
/// Dashboard 发布检查的稳定 wire 状态。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DashboardDesktopReleaseStatusView {
Unchecked,
Aligned,
ReviewRequired,
CheckFailed,
}
impl From<domain::DesktopReleaseStatus> for DashboardDesktopReleaseStatusView {
fn from(status: domain::DesktopReleaseStatus) -> Self {
match status {
domain::DesktopReleaseStatus::Unchecked => Self::Unchecked,
domain::DesktopReleaseStatus::Current => Self::Aligned,
domain::DesktopReleaseStatus::UpdateAvailable => Self::ReviewRequired,
domain::DesktopReleaseStatus::Failed => Self::CheckFailed,
}
}
}
/// Dashboard 汇总响应数据。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DashboardDataView {
pub cards: DashboardCardsView,
pub trend: TrendData,
pub health_timeline: HealthTimelineView,
pub wire_profiles: Vec<DashboardWireProfileView>,
pub account_usage: Vec<DashboardAccountUsageView>,
pub usage_records: Vec<UsageRecordView>,
pub pool_summary: DashboardPoolSummaryView,
pub capacity_info: DashboardCapacityInfoView,
pub rotation_strategy: String,
}
/// 用量汇总响应数据。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageSummaryView {
pub total_requests: String,
pub input_tokens: String,
pub output_tokens: String,
pub cached_tokens: String,
pub cache_write_tokens: String,
pub total_tokens: String,
pub average_latency_ms: String,
pub logical_requests: RequestMetricsView,
pub attempts: AttemptMetricsView,
}
/// 洞察健康趋势点。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewHealthPointView {
pub bucket: DateTime<Utc>,
pub label: String,
pub success_requests: u64,
pub failed_requests: u64,
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub error_rate: f64,
}
/// 洞察健康摘要。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewHealthView {
pub total_requests: u64,
pub success_requests: u64,
pub failed_requests: u64,
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub success_rate: f64,
pub request_change_rate: Option<f64>,
pub success_rate_change: Option<f64>,
pub points: Vec<OverviewHealthPointView>,
}
/// 洞察性能趋势点。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewPerformancePointView {
pub bucket: DateTime<Utc>,
pub label: String,
pub latency_p50_ms: Option<f64>,
pub latency_p95_ms: Option<f64>,
pub latency_p99_ms: Option<f64>,
pub first_token_p50_ms: Option<f64>,
pub first_token_p95_ms: Option<f64>,
pub first_token_p99_ms: Option<f64>,
}
/// 洞察性能摘要。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewPerformanceView {
pub latency_p50_ms: Option<f64>,
pub latency_p95_ms: Option<f64>,
pub latency_p99_ms: Option<f64>,
pub first_token_p50_ms: Option<f64>,
pub first_token_p95_ms: Option<f64>,
pub first_token_p99_ms: Option<f64>,
pub latency_coverage: f64,
pub first_token_coverage: f64,
pub points: Vec<OverviewPerformancePointView>,
}
/// 洞察成本趋势点。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewCostPointView {
pub bucket: DateTime<Utc>,
pub label: String,
pub input_tokens: u64,
pub output_tokens: u64,
pub cached_tokens: u64,
pub total_tokens: u64,
pub estimated_cost: Option<String>,
pub standard_cost: Option<String>,
pub cached_token_rate: f64,
pub cache_hit_request_rate: Option<f64>,
}
/// 洞察成本摘要。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OverviewCostView {
pub estimated_cost: Option<String>,
pub standard_cost: Option<String>,
pub cost_per_request: Option<String>,
pub tokens_per_request: f64,
pub cached_token_rate: f64,
pub cache_hit_request_rate: Option<f64>,
pub input_tokens: u64,
pub output_tokens: u64,
pub cached_tokens: u64,
pub total_tokens: u64,
pub points: Vec<OverviewCostPointView>,
pub costs: Vec<CostView>,
pub coverage: CostCoverageView,
}
/// Provider 维度洞察摘要。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ProviderOverviewView {
pub provider: String,
pub request_count: u64,
pub attempt_count: u64,
pub failure_count: u64,
pub total_tokens: u64,
}
/// 用量洞察总响应。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageInsightsOverviewView {
pub granularity: String,
pub health: OverviewHealthView,
pub performance: OverviewPerformanceView,
pub cost: OverviewCostView,
pub attempts: AttemptMetricsView,
pub providers: Vec<ProviderOverviewView>,
}
/// 诊断聚合项目。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DiagnosticItemView {
pub key: String,
pub name: String,
pub request_count: u64,
pub success_count: u64,
pub error_count: u64,
pub error_rate: f64,
pub request_share: f64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub estimated_cost: Option<String>,
pub attempt_count: u64,
pub total_tokens: u64,
}
/// 诊断聚合响应。
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DiagnosticsView {
pub dimension: String,
pub items: Vec<DiagnosticItemView>,
}
/// 运维错误项目。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OpsErrorView {
pub id: String,
pub request_id: Option<String>,
pub client_api_key_id: Option<String>,
pub kind: String,
pub provider: Option<String>,
pub authentication_kind: Option<String>,
pub account_id: Option<String>,
pub route: String,
pub model: Option<String>,
pub client_status_code: Option<i64>,
pub upstream_status_code: Option<i64>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub failure_class: String,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub message: String,
pub metadata: OpsErrorMetadataView,
pub created_at: DateTime<Utc>,
pub created_at_display: String,
}
/// 运维错误安全元数据;只保留可查询的标识,不回显秘密材料。
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OpsErrorMetadataView {
pub source: String,
pub component: String,
pub attempt_id: Option<String>,
pub account_label: Option<String>,
}
@@ -0,0 +1,39 @@
//! Admin API 的展示层辅助:数值格式化。
//!
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
/// 千分位格式化。
#[must_use]
pub fn format_number(value: u64) -> String {
let text = value.to_string();
let mut output = String::with_capacity(text.len() + text.len() / 3);
for (index, character) in text.chars().rev().enumerate() {
if index > 0 && index % 3 == 0 {
output.push(',');
}
output.push(character);
}
output.chars().rev().collect()
}
/// 紧凑格式化:≥1000 时用 K/M/B/T/P 后缀,否则千分位。
#[must_use]
pub fn format_compact_number(value: u64) -> String {
if value < 1_000 {
return format_number(value);
}
for (suffix, threshold) in [
("P", 1_000_000_000_000_000_u64),
("T", 1_000_000_000_000_u64),
("B", 1_000_000_000_u64),
("M", 1_000_000_u64),
("K", 1_000_u64),
] {
if value >= threshold {
let scaled = value as f64 / threshold as f64;
return format!("{scaled:.1}{suffix}").replace(".0", "");
}
}
format_number(value)
}
@@ -72,10 +72,7 @@ impl UpdateRuntimeSettingsRequest {
if i64::try_from(self.request_interval_ms).is_err() {
return Err(WireValidationError::new("requestIntervalMs"));
}
if !matches!(
self.rotation_strategy.as_str(),
"smart" | "quota_reset_priority" | "round_robin" | "sticky"
) {
if RotationStrategy::parse(&self.rotation_strategy).is_none() {
return Err(WireValidationError::new("rotationStrategy"));
}
Ok(())
@@ -91,7 +88,7 @@ impl UpdateRuntimeSettingsRequest {
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
request_interval_ms: self.request_interval_ms,
rotation_strategy: parse_rotation_strategy(&self.rotation_strategy)
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
usage_retention_days: u32::try_from(self.usage_retention_days)
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
@@ -111,7 +108,7 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
refresh_concurrency: u64::from(settings.refresh_concurrency),
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
request_interval_ms: settings.request_interval_ms,
rotation_strategy: rotation_strategy_name(settings.rotation_strategy).to_owned(),
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
usage_retention_days: u64::from(settings.usage_retention_days),
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
audit_retention_days: u64::from(settings.audit_retention_days),
@@ -319,25 +316,6 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
.collect()
}
fn parse_rotation_strategy(value: &str) -> Option<RotationStrategy> {
match value {
"smart" => Some(RotationStrategy::Smart),
"quota_reset_priority" => Some(RotationStrategy::QuotaResetPriority),
"round_robin" => Some(RotationStrategy::RoundRobin),
"sticky" => Some(RotationStrategy::Sticky),
_ => None,
}
}
const fn rotation_strategy_name(strategy: RotationStrategy) -> &'static str {
match strategy {
RotationStrategy::Smart => "smart",
RotationStrategy::QuotaResetPriority => "quota_reset_priority",
RotationStrategy::RoundRobin => "round_robin",
RotationStrategy::Sticky => "sticky",
}
}
fn valid_model_name(value: &str, max_len: usize) -> bool {
!value.is_empty()
&& value.len() <= max_len
+24 -2
View File
@@ -9,7 +9,7 @@ use std::sync::Arc;
use std::time::Duration;
use axum::Router;
use axum::http::{HeaderName, HeaderValue, Method, StatusCode};
use axum::http::{HeaderName, HeaderValue, Method, Request, StatusCode};
use axum::routing::get;
use gateway_admin::AdminServices;
use gateway_core::engine::execution::ExecutionService;
@@ -166,10 +166,32 @@ pub fn initialize(
Duration::from_secs(seconds),
));
}
// Trace 必须在 SetRequestId 内侧,span 才能捕获本服务生成的 request_id;
// 默认 DEBUG span 会被 info 日志过滤器丢弃,这里显式用 info_span。
let trace_layer = TraceLayer::new_for_http().make_span_with({
let request_id_header = request_id_header.clone();
move |request: &Request<_>| {
let request_id = request
.headers()
.get(&request_id_header)
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
tracing::info_span!(
"request",
request_id = %request_id,
method = %request.method().as_str(),
uri = %request
.uri()
.path_and_query()
.map(axum::http::uri::PathAndQuery::as_str)
.unwrap_or_default(),
)
}
});
let router = router
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
.layer(trace_layer)
.layer(SetRequestIdLayer::new(request_id_header, MakeRequestUuid))
.layer(TraceLayer::new_for_http())
.with_state(state);
Ok(ApiBundle { router })
}
+12 -15
View File
@@ -7,8 +7,8 @@ use axum::{
response::{IntoResponse, Response},
};
use gateway_core::{
engine::{EngineError, UpstreamSendState},
error::{GatewayError, GatewayErrorKind, ProviderErrorKind},
engine::EngineError,
error::{GatewayError, GatewayErrorKind},
};
use serde_json::{Value, json};
@@ -72,19 +72,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
#[must_use]
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
match error {
EngineError::Provider(error)
if error.kind() == ProviderErrorKind::Unavailable
&& error.send_state() == UpstreamSendState::NotSent =>
{
let gateway = GatewayError::new(
GatewayErrorKind::NoAvailableProvider,
"no upstream provider is currently available for this request",
);
match error.client_visible_upstream_error() {
Some(upstream) => gateway.with_client_visible_upstream_error(upstream.clone()),
None => gateway,
}
}
EngineError::Provider(error) => GatewayError::from_provider(error),
EngineError::Cancelled => {
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
@@ -185,6 +172,16 @@ pub const fn gateway_error_contract(
"server_error",
"no_available_provider",
),
GatewayErrorKind::AccountCapacityUnavailable => (
StatusCode::SERVICE_UNAVAILABLE,
"server_error",
"account_capacity_unavailable",
),
GatewayErrorKind::ProviderInfrastructureUnavailable => (
StatusCode::SERVICE_UNAVAILABLE,
"server_error",
"provider_infrastructure_unavailable",
),
GatewayErrorKind::RateLimited => (
StatusCode::TOO_MANY_REQUESTS,
"rate_limit_error",
@@ -470,6 +470,15 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
| "x-openai-project"
// installation ID 始终由当前 lease 重建。
| "x-codex-installation-id"
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent
// 不能透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
| "originator"
| "user-agent"
// 设备 attestation/integrity 头只能由官方客户端或网关自身生成,
// 客户端注入的 x-oai-attestation / X-OAI-IS 不得透传上游。
| "x-oai-attestation"
| "x-oai-is"
| "x-oai-is-update"
)
}
@@ -340,7 +340,7 @@ mod actions {
upstream_status: Some(429),
upstream_content_type: Some("application/json".to_owned()),
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
account_status: AccountStatus::Active,
account_status: AccountStatus::QuotaExhausted,
},
]
.map(|event| AccountConnectionTestEvent::from(event).data);
@@ -374,7 +374,7 @@ mod actions {
"upstreamStatus": 429,
"upstreamContentType": "application/json",
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
"accountStatus": "active"
"accountStatus": "quota_exhausted"
}),
]
);
+41 -7
View File
@@ -23,9 +23,9 @@ use gateway_admin::{
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
},
observability::{
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsErrorPage,
OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter, UsageOverview,
UsagePage, UsageQuery, UsageRecord,
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
UsageOverview, UsagePage, UsageQuery, UsageRecord,
},
provider_credentials::{
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
@@ -79,6 +79,10 @@ pub(super) struct AdminTestFixture {
pub auth: Arc<MemoryAuthStore>,
pub settings: Arc<MemorySettingsStore>,
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
pub usage_detail: Arc<Mutex<Option<UsageDetail>>>,
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
}
impl AdminTestFixture {
@@ -92,8 +96,16 @@ impl AdminTestFixture {
let settings = Arc::new(MemorySettingsStore::new(api_key));
let client_keys = Arc::new(MemoryClientKeyStore);
let usage_records = Arc::new(Mutex::new(Vec::new()));
let usage_detail = Arc::new(Mutex::new(None));
let diagnostics = Arc::new(Mutex::new(Vec::new()));
let ops_errors = Arc::new(Mutex::new(Vec::new()));
let dashboard_observation = Arc::new(Mutex::new(None));
let unused = Arc::new(UnusedStore {
usage_records: Arc::clone(&usage_records),
usage_detail: Arc::clone(&usage_detail),
diagnostics: Arc::clone(&diagnostics),
ops_errors: Arc::clone(&ops_errors),
dashboard_observation: Arc::clone(&dashboard_observation),
});
let stores = AdminStorePorts::new(
unused.clone(),
@@ -101,6 +113,7 @@ impl AdminTestFixture {
client_keys.clone(),
unused,
settings.clone(),
gateway_admin::ports::backup::BackupStorePorts::disabled(),
);
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
Arc::new(UnusedProvider::new("openai")),
@@ -125,6 +138,10 @@ impl AdminTestFixture {
auth,
settings,
usage_records,
usage_detail,
diagnostics,
ops_errors,
dashboard_observation,
}
}
@@ -392,6 +409,10 @@ impl ClientKeyStore for MemoryClientKeyStore {
struct UnusedStore {
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
usage_detail: Arc<Mutex<Option<UsageDetail>>>,
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
ops_errors: Arc<Mutex<Vec<OpsError>>>,
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
}
#[async_trait]
@@ -503,7 +524,11 @@ impl AccountStore for UnusedStore {
#[async_trait]
impl ObservabilityStore for UnusedStore {
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
Err(unavailable("dashboard"))
self.dashboard_observation
.lock()
.expect("dashboard observation")
.clone()
.ok_or_else(|| unavailable("dashboard"))
}
async fn dashboard_trend(&self, _: TimeRange) -> AdminStoreResult<Vec<RequestMetricPoint>> {
@@ -537,7 +562,11 @@ impl ObservabilityStore for UnusedStore {
}
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
Err(unavailable("usage detail"))
self.usage_detail
.lock()
.expect("usage detail")
.clone()
.ok_or_else(|| unavailable("usage detail"))
}
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
@@ -550,11 +579,16 @@ impl ObservabilityStore for UnusedStore {
_: UsageFilter,
_: DiagnosticDimension,
) -> AdminStoreResult<Vec<DiagnosticObservation>> {
Err(unavailable("usage diagnostics"))
Ok(self.diagnostics.lock().expect("diagnostics").clone())
}
async fn list_ops_errors(&self, _: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
Err(unavailable("ops errors"))
let items = self.ops_errors.lock().expect("ops errors").clone();
Ok(OpsErrorPage {
total: items.len() as u64,
items,
next_cursor: None,
})
}
}
@@ -1,459 +0,0 @@
mod query {
use gateway_api::admin::observability::{
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
parse_attempt_index, parse_datetime, parse_status,
};
use serde_json::json;
#[test]
fn dashboard_query_should_parse_terminal_trend_kinds() {
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
}
#[test]
fn dashboard_query_should_reject_unknown_trend_kind() {
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
}
#[test]
fn usage_query_should_bound_page_size_and_cursor() {
let query: UsageQuery = serde_json::from_value(json!({
"page": 2,
"pageSize": 100,
"cursor": "opaque"
}))
.unwrap();
assert_eq!(query.validate_page().unwrap(), (2, 100));
assert!(query.validate_cursor().is_ok());
}
#[test]
fn usage_query_should_reject_zero_page() {
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
assert_eq!(query.validate_page().unwrap_err().field(), "page");
}
#[test]
fn ops_query_should_reject_page_size_above_terminal_limit() {
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
}
#[test]
fn diagnostics_query_should_keep_wire_dimension_name() {
let query: DiagnosticsQuery =
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
}
#[test]
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
assert_eq!(
parse_attempt_index(Some(0)).unwrap_err().field(),
"attemptIndex"
);
assert_eq!(
parse_datetime(Some("not-a-time")).unwrap_err().field(),
"timeRange"
);
}
}
mod response {
use chrono::{TimeZone, Utc};
use gateway_admin::model::observability::DesktopReleaseStatus;
use gateway_api::admin::observability::{
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
DashboardWireProfileView, DashboardWireTargetView, PageData, PageMeta, TokenDetailsView,
TrendData, TrendKind, TrendPointView, TrendSummaryView,
};
use serde_json::json;
#[test]
fn usage_page_should_keep_terminal_camel_case_shape() {
let data = PageData {
items: vec![json!({"id": "request_1"})],
page: PageMeta::new(1, 50, 1, 1),
next_cursor: Some("cursor".to_owned()),
};
let value = serde_json::to_value(data).unwrap();
assert_eq!(value["page"]["pageSize"], 50);
assert_eq!(value["nextCursor"], "cursor");
}
#[test]
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
let value = serde_json::to_value(DashboardWireProfileView {
provider: "xai".to_owned(),
product: "Grok Build".to_owned(),
version: "0.2.106".to_owned(),
build: None,
target: DashboardWireTargetView {
os_type: "linux".to_owned(),
os_version: "—".to_owned(),
arch: "x86_64".to_owned(),
terminal: "headless".to_owned(),
},
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
attributes: vec![DashboardWireAttributeView {
label: "客户端标识".to_owned(),
value: "grok-shell".to_owned(),
}],
verified_at: None,
release: None,
})
.expect("dashboard profile");
assert_eq!(value["provider"], "xai");
assert_eq!(value["version"], "0.2.106");
assert_eq!(value["attributes"][0]["label"], "客户端标识");
assert!(value.get("release").is_none());
assert!(value.get("verifiedAt").is_none());
}
#[test]
fn dashboard_account_usage_should_keep_daily_request_timeline() {
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
let value = serde_json::to_value(DashboardAccountUsageView {
id: "account_1".to_owned(),
provider: "xai".to_owned(),
authentication_kind: "oauth".to_owned(),
email: "account@example.com".to_owned(),
plan_type: Some("free".to_owned()),
tokens: "—".to_owned(),
request_count: 3,
request_buckets: vec![DashboardAccountRequestBucketView {
bucket_start,
request_count: 3,
}],
quota_used_percent: None,
last_used: "刚刚".to_owned(),
})
.expect("dashboard account usage");
assert_eq!(value["requestCount"], 3);
assert_eq!(
value["requestBuckets"][0]["bucketStart"],
"1970-01-01T00:00:00Z"
);
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
}
#[test]
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
let now = Utc.timestamp_opt(0, 0).single().unwrap();
let data = TrendData {
kind: TrendKind::Usage,
points: vec![TrendPointView {
time: "08:00".to_owned(),
bucket: now,
label: "01-01 08:00".to_owned(),
requests: "1".to_owned(),
requests_value: 1,
input_tokens: "2".to_owned(),
input_tokens_value: 2,
output_tokens: "3".to_owned(),
output_tokens_value: 3,
cached_tokens: "0".to_owned(),
cached_tokens_value: 0,
cache_hit_rate_value: 0.0,
tokens_value: 5,
errors: "0".to_owned(),
errors_value: 0,
latency: "1 ms".to_owned(),
latency_value: Some(1),
first_token_latency: "1 ms".to_owned(),
first_token_latency_value: Some(1),
max_latency: "1 ms".to_owned(),
max_latency_value: Some(1),
min_latency: "1 ms".to_owned(),
min_latency_value: Some(1),
success_rate: "100.0%".to_owned(),
success_rate_value: Some(100.0),
}],
summary: vec![TrendSummaryView {
label: "输入".to_owned(),
value: "2".to_owned(),
ratio: None,
}],
};
let value = serde_json::to_value(data).unwrap();
assert_eq!(value["kind"], "usage");
assert_eq!(value["points"][0]["requestsValue"], 1);
}
#[test]
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
let coverage = CostCoverageView {
known: 1,
partial: 0,
unknown: 0,
not_billable: 0,
};
let token_details = TokenDetailsView {
input_tokens: Some(1),
output_tokens: Some(2),
cached_tokens: None,
cache_write_tokens: None,
reasoning_tokens: None,
image_input_tokens: None,
image_output_tokens: None,
total_tokens: Some(3),
input_tokens_display: "1".to_owned(),
output_tokens_display: "2".to_owned(),
cached_tokens_display: "-".to_owned(),
cache_write_tokens_display: "-".to_owned(),
reasoning_tokens_display: "-".to_owned(),
image_input_tokens_display: "-".to_owned(),
image_output_tokens_display: "-".to_owned(),
total_tokens_display: "3".to_owned(),
};
let cursor = CursorWire {
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
stable_id: "request_1".to_owned(),
};
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
assert!(value.to_string().contains("known"));
assert!(!value.to_string().contains("secret"));
}
#[test]
fn billing_view_should_preserve_the_original_detail_contract() {
let value = serde_json::to_value(BillingView {
input_amount_display: "$0.03".to_owned(),
output_amount_display: "$0.00".to_owned(),
cache_read_amount_display: "$0.14".to_owned(),
cache_write_amount_display: "$0.00".to_owned(),
standard_amount_display: "$0.17".to_owned(),
total_amount_display: "$0.17".to_owned(),
input_price_display: "$10.0000 / 1M Token".to_owned(),
output_price_display: "$60.0000 / 1M Token".to_owned(),
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
service_tier_display: "Fast".to_owned(),
multiplier_display: "1.00x".to_owned(),
})
.expect("billing view");
assert_eq!(value["inputAmountDisplay"], "$0.03");
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
assert_eq!(value["serviceTierDisplay"], "Fast");
assert_eq!(value["multiplierDisplay"], "1.00x");
}
#[test]
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
for (domain, expected) in [
(DesktopReleaseStatus::Unchecked, "unchecked"),
(DesktopReleaseStatus::Current, "aligned"),
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
(DesktopReleaseStatus::Failed, "check_failed"),
] {
let status = DashboardDesktopReleaseStatusView::from(domain);
assert_eq!(serde_json::to_value(status).unwrap(), expected);
}
}
}
#[tokio::test]
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use super::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records?outcome=provider_future_state")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_usage_other_outcome")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn usage_route_should_expose_image_and_websocket_facts() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use super::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
fixture
.usage_records
.lock()
.expect("usage records")
.push(UsageRecord {
id: "request_endpoint".to_owned(),
client_api_key_ref: "key_endpoint".to_owned(),
config_revision: 1,
protocol: "openai".to_owned(),
operation: "generate".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "grok-4.5".to_owned(),
provider_kind: Some("xai".to_owned()),
provider_account_ref: None,
provider_account_name: None,
provider_account_email: None,
provider_account_authentication_kind: Some("oauth".to_owned()),
upstream_model_id: Some("grok-4.5".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
http_version: Some("h2".to_owned()),
websocket_pool: Some("reuse".to_owned()),
service_tier: Some("priority".to_owned()),
provider_metadata_json: Some(
serde_json::json!({
"effectiveModel": "grok-4.5",
"requestSummary": {"inputItemsCount": 2},
"transport": "must-not-overwrite-core",
})
.to_string(),
),
attempt_count: 1,
upstream_send_state: "sent".to_owned(),
downstream_committed_at: None,
outcome: RequestOutcome::Succeeded,
client_status_code: Some(200),
upstream_status_code: Some(200),
client_response_id: Some("resp_usage_metadata".to_owned()),
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
upstream_response_id: None,
error_kind: None,
provider_error_code: None,
error_message: None,
retry_after_ms: None,
input_tokens: Some(1),
output_tokens: Some(1),
cached_tokens: Some(0),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
image_input_tokens: Some(31),
image_output_tokens: Some(9),
total_tokens: Some(2),
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: None,
transport_decision_wait_ms: Some(7),
connect_ms: Some(11),
headers_ms: Some(13),
first_event_ms: Some(17),
first_reasoning_ms: Some(19),
first_text_ms: Some(23),
first_token_ms: Some(19),
provider_processing_ms: Some(29),
latency_ms: Some(31),
client_ip: Some("127.0.0.1".to_owned()),
user_agent: Some("usage-metadata-test".to_owned()),
reasoning_effort: Some("max".to_owned()),
reasoning_preset: Some("ultra".to_owned()),
request_kind: Some("review".to_owned()),
subagent_kind: Some("worker".to_owned()),
compact: true,
image_generation_requested: true,
image_generation_succeeded: Some(true),
started_at: Utc::now(),
deadline_at: Utc::now(),
completed_at: Some(Utc::now()),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_usage_endpoint")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("usage response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
assert_eq!(
serde_json::json!({
"route": value["data"]["items"][0]["route"],
"serviceTier": value["data"]["items"][0]["serviceTier"],
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
"websocketPool": value["data"]["items"][0]["metadata"]["websocketPool"],
"imageGenerationRequested": value["data"]["items"][0]["metadata"]["imageGenerationRequested"],
"imageGenerationSucceeded": value["data"]["items"][0]["metadata"]["imageGenerationSucceeded"],
"requestedModel": value["data"]["items"][0]["metadata"]["requestedModel"],
"upstreamModel": value["data"]["items"][0]["metadata"]["upstreamModel"],
"clientIp": value["data"]["items"][0]["metadata"]["clientIp"],
"userAgent": value["data"]["items"][0]["metadata"]["userAgent"],
"reasoningEffort": value["data"]["items"][0]["metadata"]["reasoningEffort"],
"reasoningPreset": value["data"]["items"][0]["metadata"]["reasoningPreset"],
"requestKind": value["data"]["items"][0]["metadata"]["requestKind"],
"subagentKind": value["data"]["items"][0]["metadata"]["subagentKind"],
"compact": value["data"]["items"][0]["metadata"]["compact"],
"transport": value["data"]["items"][0]["metadata"]["transport"],
"transportDecisionWaitMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["transportDecisionWaitMs"],
"wsConnectMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["wsConnectMs"],
"firstReasoningMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstReasoningMs"],
"firstTextMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTextMs"],
"firstTokenMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTokenMs"],
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
}),
serde_json::json!({
"route": "/v1/responses",
"serviceTier": "priority",
"authenticationKind": "oauth",
"imageInputTokens": 31,
"imageOutputTokens": 9,
"websocketPool": {"kind": "reuse"},
"imageGenerationRequested": true,
"imageGenerationSucceeded": true,
"requestedModel": "grok-4.5",
"upstreamModel": "grok-4.5",
"clientIp": "127.0.0.1",
"userAgent": "usage-metadata-test",
"reasoningEffort": "max",
"reasoningPreset": "ultra",
"requestKind": "review",
"subagentKind": "worker",
"compact": true,
"transport": "http_sse",
"transportDecisionWaitMs": 7,
"wsConnectMs": 11,
"firstReasoningMs": 19,
"firstTextMs": 23,
"firstTokenMs": 19,
"effectiveModel": "grok-4.5",
"requestSummary": {"inputItemsCount": 2},
})
);
}
@@ -0,0 +1,4 @@
//! Admin 观测 wire/路由的固定合同测试。
mod query;
mod response;
@@ -0,0 +1,64 @@
//! 查询 wire 校验的固定合同测试。
use gateway_api::admin::observability::{
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
parse_attempt_index, parse_datetime, parse_status,
};
use serde_json::json;
#[test]
fn dashboard_query_should_parse_terminal_trend_kinds() {
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
}
#[test]
fn dashboard_query_should_reject_unknown_trend_kind() {
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
}
#[test]
fn usage_query_should_bound_page_size_and_cursor() {
let query: UsageQuery = serde_json::from_value(json!({
"page": 2,
"pageSize": 100,
"cursor": "opaque"
}))
.unwrap();
assert_eq!(query.validate_page().unwrap(), (2, 100));
assert!(query.validate_cursor().is_ok());
}
#[test]
fn usage_query_should_reject_zero_page() {
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
assert_eq!(query.validate_page().unwrap_err().field(), "page");
}
#[test]
fn ops_query_should_reject_page_size_above_terminal_limit() {
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
}
#[test]
fn diagnostics_query_should_keep_wire_dimension_name() {
let query: DiagnosticsQuery =
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
}
#[test]
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
assert_eq!(
parse_attempt_index(Some(0)).unwrap_err().field(),
"attemptIndex"
);
assert_eq!(
parse_datetime(Some("not-a-time")).unwrap_err().field(),
"timeRange"
);
}
@@ -0,0 +1,848 @@
//! 响应 DTO 与固定 wire 形状的合同测试。
use chrono::{TimeZone, Utc};
use gateway_admin::model::observability::DesktopReleaseStatus;
use gateway_api::admin::PageMeta;
use gateway_api::admin::observability::{
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData,
TrendKind, TrendPointView, TrendSummaryView,
};
use serde_json::json;
#[test]
fn usage_page_should_keep_terminal_camel_case_shape() {
let data = PageData {
items: vec![json!({"id": "request_1"})],
page: PageMeta::new(1, 50, 1, 1),
next_cursor: Some("cursor".to_owned()),
};
let value = serde_json::to_value(data).unwrap();
assert_eq!(value["page"]["pageSize"], 50);
assert_eq!(value["nextCursor"], "cursor");
}
#[test]
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
let value = serde_json::to_value(DashboardWireProfileView {
provider: "xai".to_owned(),
product: "Grok Build".to_owned(),
version: "0.2.106".to_owned(),
build: None,
target: DashboardWireTargetView {
os_type: "linux".to_owned(),
os_version: "—".to_owned(),
arch: "x86_64".to_owned(),
terminal: "headless".to_owned(),
},
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
attributes: vec![DashboardWireAttributeView {
label: "客户端标识".to_owned(),
value: "grok-shell".to_owned(),
}],
verified_at: None,
release: None,
})
.expect("dashboard profile");
assert_eq!(value["provider"], "xai");
assert_eq!(value["version"], "0.2.106");
assert_eq!(value["attributes"][0]["label"], "客户端标识");
assert!(value.get("release").is_none());
assert!(value.get("verifiedAt").is_none());
}
#[test]
fn dashboard_account_usage_should_keep_daily_request_timeline() {
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
let value = serde_json::to_value(DashboardAccountUsageView {
id: "account_1".to_owned(),
provider: "xai".to_owned(),
authentication_kind: "oauth".to_owned(),
email: "account@example.com".to_owned(),
plan_type: Some("free".to_owned()),
tokens: "—".to_owned(),
request_count: 3,
request_buckets: vec![DashboardAccountRequestBucketView {
bucket_start,
request_count: 3,
}],
quota_used_percent: None,
last_used: "刚刚".to_owned(),
})
.expect("dashboard account usage");
assert_eq!(value["requestCount"], 3);
assert_eq!(
value["requestBuckets"][0]["bucketStart"],
"1970-01-01T00:00:00Z"
);
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
}
#[test]
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
let now = Utc.timestamp_opt(0, 0).single().unwrap();
let data = TrendData {
kind: TrendKind::Usage,
points: vec![TrendPointView {
time: "08:00".to_owned(),
bucket: now,
label: "01-01 08:00".to_owned(),
requests: "1".to_owned(),
requests_value: 1,
input_tokens: "2".to_owned(),
input_tokens_value: 2,
output_tokens: "3".to_owned(),
output_tokens_value: 3,
cached_tokens: "0".to_owned(),
cached_tokens_value: 0,
cache_hit_rate_value: 0.0,
tokens_value: 5,
errors: "0".to_owned(),
errors_value: 0,
latency: "1 ms".to_owned(),
latency_value: Some(1),
max_latency: "1 ms".to_owned(),
max_latency_value: Some(1),
min_latency: "1 ms".to_owned(),
min_latency_value: Some(1),
success_rate: "100.0%".to_owned(),
success_rate_value: Some(100.0),
}],
summary: vec![TrendSummaryView {
label: "输入".to_owned(),
value: "2".to_owned(),
ratio: None,
}],
};
let value = serde_json::to_value(data).unwrap();
assert_eq!(value["kind"], "usage");
assert_eq!(value["points"][0]["requestsValue"], 1);
}
#[test]
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
let coverage = CostCoverageView {
known: 1,
partial: 0,
unknown: 0,
not_billable: 0,
};
let token_details = TokenDetailsView {
input_tokens: Some(1),
output_tokens: Some(2),
cached_tokens: None,
cache_write_tokens: None,
reasoning_tokens: None,
image_input_tokens: None,
image_output_tokens: None,
total_tokens: Some(3),
input_tokens_display: "1".to_owned(),
output_tokens_display: "2".to_owned(),
cached_tokens_display: "-".to_owned(),
cache_write_tokens_display: "-".to_owned(),
reasoning_tokens_display: "-".to_owned(),
image_input_tokens_display: "-".to_owned(),
image_output_tokens_display: "-".to_owned(),
total_tokens_display: "3".to_owned(),
};
let cursor = CursorWire {
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
stable_id: "request_1".to_owned(),
};
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
assert!(value.to_string().contains("known"));
assert!(!value.to_string().contains("secret"));
}
#[test]
fn billing_view_should_preserve_the_original_detail_contract() {
let value = serde_json::to_value(BillingView {
input_amount_display: "$0.03".to_owned(),
output_amount_display: "$0.00".to_owned(),
cache_read_amount_display: "$0.14".to_owned(),
cache_write_amount_display: "$0.00".to_owned(),
standard_amount_display: "$0.17".to_owned(),
total_amount_display: "$0.17".to_owned(),
input_price_display: "$10.0000 / 1M Token".to_owned(),
output_price_display: "$60.0000 / 1M Token".to_owned(),
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
service_tier_display: "Fast".to_owned(),
multiplier_display: "1.00x".to_owned(),
})
.expect("billing view");
assert_eq!(value["inputAmountDisplay"], "$0.03");
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
assert_eq!(value["serviceTierDisplay"], "Fast");
assert_eq!(value["multiplierDisplay"], "1.00x");
}
#[test]
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
for (domain, expected) in [
(DesktopReleaseStatus::Unchecked, "unchecked"),
(DesktopReleaseStatus::Current, "aligned"),
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
(DesktopReleaseStatus::Failed, "check_failed"),
] {
let status = DashboardDesktopReleaseStatusView::from(domain);
assert_eq!(serde_json::to_value(status).unwrap(), expected);
}
}
#[tokio::test]
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
requests: RequestMetrics::default(),
attempts: AttemptMetrics::default(),
provider_accounts: AccountPoolMetrics {
total: 807,
enabled: 804,
unavailable: 792,
active: 15,
rate_limited: 0,
expired: 2,
invalid: 0,
quota_exhausted: 785,
refreshing: None,
disabled: 3,
banned: 2,
},
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_dashboard_account_summary")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("dashboard response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
assert_eq!(status, StatusCode::OK, "{value}");
assert_eq!(
(
&value["data"]["cards"]["credentials"]["totalValue"],
&value["data"]["cards"]["credentials"]["availableValue"],
&value["data"]["cards"]["credentials"]["unavailableValue"],
),
(
&serde_json::json!(807),
&serde_json::json!(15),
&serde_json::json!(792)
),
);
}
#[tokio::test]
async fn usage_detail_should_keep_attempt_snapshot_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{UsageAttempt, UsageDetail};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
fixture
.usage_detail
.lock()
.expect("usage detail")
.replace(UsageDetail {
request: usage_record_with_account(
"req_detail",
"acct_snap_a",
"Snapshot Alpha",
"alpha@example.invalid",
"oauth",
now,
),
attempts: vec![UsageAttempt {
source: "ops_event".to_owned(),
id: "ops_detail".to_owned(),
attempt_index: 1,
component: "routing".to_owned(),
operation: "fallback".to_owned(),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_snap_b".to_owned()),
provider_account_name: Some("Snapshot Beta".to_owned()),
provider_account_email: Some("beta@example.invalid".to_owned()),
provider_account_authentication_kind: Some("api_key".to_owned()),
upstream_model_id: Some("upstream-b".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
upstream_send_state: Some("sent".to_owned()),
outcome: gateway_admin::model::observability::RequestOutcome::Failed,
downstream_committed: false,
status_code: Some(429),
provider_error_code: Some("rate_limit".to_owned()),
failure_kind: Some("rate_limited".to_owned()),
retry_after_ms: Some(1_000),
upstream_request_id: None,
latency_ms: Some(120),
message: Some("limited".to_owned()),
input_tokens: None,
output_tokens: None,
cached_tokens: None,
cache_write_tokens: None,
reasoning_tokens: None,
total_tokens: None,
cost_source: Some("unavailable".to_owned()),
cost_amount: None,
cost_currency: None,
occurred_at: now,
}],
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records/detail?id=req_detail")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_usage_detail_snapshot")
.body(Body::empty())
.expect("usage detail request"),
)
.await
.expect("usage detail response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("usage detail body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage detail JSON");
assert_eq!(
serde_json::json!({
"accountId": value["data"]["accountId"],
"accountName": value["data"]["accountName"],
"accountEmail": value["data"]["accountEmail"],
"authenticationKind": value["data"]["authenticationKind"],
"attemptCredentialName": value["data"]["attempts"][0]["credentialName"],
"attemptAccountId": value["data"]["attempts"][0]["accountId"],
"attemptAccountName": value["data"]["attempts"][0]["accountName"],
"attemptAccountEmail": value["data"]["attempts"][0]["accountEmail"],
"attemptAuthenticationKind": value["data"]["attempts"][0]["authenticationKind"],
}),
serde_json::json!({
"accountId": "acct_snap_a",
"accountName": "Snapshot Alpha",
"accountEmail": "alpha@example.invalid",
"authenticationKind": "oauth",
"attemptCredentialName": "Snapshot Beta",
"attemptAccountId": "acct_snap_b",
"attemptAccountName": "Snapshot Beta",
"attemptAccountEmail": "beta@example.invalid",
"attemptAuthenticationKind": "api_key",
})
);
}
#[tokio::test]
async fn ops_errors_should_keep_account_label_and_authentication_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::OpsError;
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
fixture
.ops_errors
.lock()
.expect("ops errors")
.push(OpsError {
source: "model_request".to_owned(),
event_id: "err_snapshot".to_owned(),
request_id: Some("req_err".to_owned()),
attempt_index: Some(1),
client_api_key_ref: Some("key_err".to_owned()),
component: "model_request".to_owned(),
operation: "responses".to_owned(),
endpoint: Some("/v1/responses".to_owned()),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_err".to_owned()),
provider_account_name: None,
provider_account_email: Some("err@example.invalid".to_owned()),
provider_account_authentication_kind: Some("api_key".to_owned()),
upstream_model_id: Some("upstream-err".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
failure_kind: "upstream_error".to_owned(),
client_status_code: Some(502),
upstream_status_code: Some(502),
provider_error_code: Some("upstream".to_owned()),
client_response_id: None,
upstream_request_id: None,
latency_ms: Some(90),
message: "snapshot error".to_owned(),
occurrence_count: 1,
occurred_at: Utc::now(),
stable_sort_id: "model_request:req_err".to_owned(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/operations/errors")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_ops_errors_snapshot")
.body(Body::empty())
.expect("ops errors request"),
)
.await
.expect("ops errors response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("ops errors body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("ops errors JSON");
assert_eq!(
serde_json::json!({
"provider": value["data"]["items"][0]["provider"],
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
"kind": value["data"]["items"][0]["kind"],
"accountId": value["data"]["items"][0]["accountId"],
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
"route": value["data"]["items"][0]["route"],
}),
serde_json::json!({
"provider": "openai",
"authenticationKind": "api_key",
"kind": "model_request",
"accountId": "acct_err",
"accountLabel": "err@example.invalid",
"clientStatusCode": 502,
"route": "/v1/responses",
})
);
}
#[tokio::test]
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_admin::model::observability::{CostCoverage, DiagnosticObservation};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
fixture
.diagnostics
.lock()
.expect("diagnostics")
.push(DiagnosticObservation {
key: "acct_diag".to_owned(),
name: "diag@example.invalid".to_owned(),
request_count: 2,
success_count: 2,
failure_count: 0,
attempt_count: 2,
total_tokens: 200,
average_latency_ms: Some(100),
latency_p95_ms: Some(3800),
cost_coverage: CostCoverage::default(),
costs: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/insights/diagnostics?dimension=account")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_diagnostics_snapshot")
.body(Body::empty())
.expect("diagnostics request"),
)
.await
.expect("diagnostics response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("diagnostics body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
assert_eq!(
(
&value["data"]["items"][0]["key"],
&value["data"]["items"][0]["name"],
&value["data"]["items"][0]["latencyP95Ms"],
&value["data"]["dimension"],
),
(
&serde_json::json!("acct_diag"),
&serde_json::json!("diag@example.invalid"),
&serde_json::json!(3800),
&serde_json::json!("account"),
)
);
}
fn usage_record_with_account(
id: &str,
account_id: &str,
name: &str,
email: &str,
authentication_kind: &str,
started_at: chrono::DateTime<chrono::Utc>,
) -> gateway_admin::model::observability::UsageRecord {
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
UsageRecord {
id: id.to_owned(),
client_api_key_ref: "key_detail".to_owned(),
config_revision: 1,
protocol: "openai".to_owned(),
operation: "responses".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "coding".to_owned(),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some(account_id.to_owned()),
provider_account_name: Some(name.to_owned()),
provider_account_email: Some(email.to_owned()),
provider_account_authentication_kind: Some(authentication_kind.to_owned()),
upstream_model_id: Some("upstream-model".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
http_version: Some("h2".to_owned()),
websocket_pool: None,
service_tier: None,
provider_metadata_json: None,
attempt_count: 1,
upstream_send_state: "sent".to_owned(),
downstream_committed_at: Some(started_at),
outcome: RequestOutcome::Succeeded,
client_status_code: Some(200),
upstream_status_code: Some(200),
client_response_id: None,
upstream_request_id: None,
upstream_response_id: None,
error_kind: None,
provider_error_code: None,
error_message: None,
retry_after_ms: None,
input_tokens: Some(1),
output_tokens: Some(1),
cached_tokens: Some(0),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
image_input_tokens: Some(0),
image_output_tokens: Some(0),
total_tokens: Some(2),
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: None,
transport_decision_wait_ms: None,
connect_ms: None,
headers_ms: None,
first_event_ms: None,
first_reasoning_ms: None,
first_text_ms: None,
first_token_ms: None,
provider_processing_ms: None,
latency_ms: None,
client_ip: None,
user_agent: None,
reasoning_effort: None,
reasoning_preset: None,
request_kind: None,
subagent_kind: None,
compact: false,
image_generation_requested: false,
image_generation_succeeded: None,
started_at,
deadline_at: started_at + chrono::Duration::seconds(30),
completed_at: Some(started_at),
}
}
#[tokio::test]
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records?outcome=provider_future_state")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_usage_other_outcome")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn usage_route_should_expose_image_and_websocket_facts() {
use std::str::FromStr as _;
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
UsageRecord,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let usd = |amount: &str| CurrencyCost {
currency: "USD".to_owned(),
amount: DecimalAmount::from_str(amount).expect("USD amount"),
};
fixture
.usage_records
.lock()
.expect("usage records")
.push(UsageRecord {
id: "request_endpoint".to_owned(),
client_api_key_ref: "key_endpoint".to_owned(),
config_revision: 1,
protocol: "openai".to_owned(),
operation: "generate".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: "grok-4.5".to_owned(),
provider_kind: Some("xai".to_owned()),
provider_account_ref: Some("acct_snapshot".to_owned()),
provider_account_name: Some("Snapshot Alpha".to_owned()),
provider_account_email: Some("alpha@example.invalid".to_owned()),
provider_account_authentication_kind: Some("oauth".to_owned()),
upstream_model_id: Some("grok-4.5".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
http_version: Some("h2".to_owned()),
websocket_pool: Some("reuse".to_owned()),
service_tier: Some("priority".to_owned()),
provider_metadata_json: Some(
serde_json::json!({
"effectiveModel": "grok-4.5",
"requestSummary": {"inputItemsCount": 2},
"transport": "must-not-overwrite-core",
})
.to_string(),
),
attempt_count: 1,
upstream_send_state: "sent".to_owned(),
downstream_committed_at: None,
outcome: RequestOutcome::Succeeded,
client_status_code: Some(200),
upstream_status_code: Some(200),
client_response_id: Some("resp_usage_metadata".to_owned()),
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
upstream_response_id: None,
error_kind: None,
provider_error_code: None,
error_message: None,
retry_after_ms: None,
input_tokens: Some(1),
output_tokens: Some(1),
cached_tokens: Some(0),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
image_input_tokens: Some(31),
image_output_tokens: Some(9),
total_tokens: Some(2),
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: Some(UsageBilling::Calculated(Box::new(
CalculatedBillingBreakdown {
input_amount: usd("0.03"),
output_amount: usd("0.07"),
cache_read_amount: usd("0.00"),
cache_write_amount: usd("0.00"),
standard_amount: usd("0.10"),
total_amount: usd("0.10"),
input_price_per_million: usd("10.0000"),
output_price_per_million: usd("60.0000"),
cache_read_price_per_million: usd("1.0000"),
cache_write_price_per_million: usd("12.5000"),
service_tier: Some("priority".to_owned()),
multiplier_percent: 100,
},
))),
transport_decision_wait_ms: Some(7),
connect_ms: Some(11),
headers_ms: Some(13),
first_event_ms: Some(17),
first_reasoning_ms: Some(19),
first_text_ms: Some(23),
first_token_ms: Some(19),
provider_processing_ms: Some(29),
latency_ms: Some(31),
client_ip: Some("127.0.0.1".to_owned()),
user_agent: Some("usage-metadata-test".to_owned()),
reasoning_effort: Some("max".to_owned()),
reasoning_preset: Some("ultra".to_owned()),
request_kind: Some("review".to_owned()),
subagent_kind: Some("worker".to_owned()),
compact: true,
image_generation_requested: true,
image_generation_succeeded: Some(true),
started_at: Utc::now(),
deadline_at: Utc::now(),
completed_at: Some(Utc::now()),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_usage_endpoint")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("usage response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
assert_eq!(
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
"$10 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
"$60 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
"$12.5 / 1M Token"
);
assert_eq!(
serde_json::json!({
"route": value["data"]["items"][0]["route"],
"serviceTier": value["data"]["items"][0]["serviceTier"],
"accountId": value["data"]["items"][0]["accountId"],
"accountName": value["data"]["items"][0]["accountName"],
"accountEmail": value["data"]["items"][0]["accountEmail"],
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
"websocketPool": value["data"]["items"][0]["websocketPool"],
"imageGenerationRequested": value["data"]["items"][0]["imageGenerationRequested"],
"imageGenerationSucceeded": value["data"]["items"][0]["imageGenerationSucceeded"],
"requestedModel": value["data"]["items"][0]["requestedModel"],
"upstreamModel": value["data"]["items"][0]["upstreamModel"],
"clientIp": value["data"]["items"][0]["clientIp"],
"userAgent": value["data"]["items"][0]["userAgent"],
"reasoningEffort": value["data"]["items"][0]["reasoningEffort"],
"reasoningPreset": value["data"]["items"][0]["reasoningPreset"],
"requestKind": value["data"]["items"][0]["requestKind"],
"subagentKind": value["data"]["items"][0]["subagentKind"],
"compact": value["data"]["items"][0]["compact"],
"transport": value["data"]["items"][0]["transport"],
"transportDecisionWaitMs": value["data"]["items"][0]["latencyDetails"]["transportDecisionWaitMs"],
"wsConnectMs": value["data"]["items"][0]["latencyDetails"]["wsConnectMs"],
"firstReasoningMs": value["data"]["items"][0]["latencyDetails"]["firstReasoningMs"],
"firstTextMs": value["data"]["items"][0]["latencyDetails"]["firstTextMs"],
"firstTokenMs": value["data"]["items"][0]["latencyDetails"]["firstTokenMs"],
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
}),
serde_json::json!({
"route": "/v1/responses",
"serviceTier": "priority",
"accountId": "acct_snapshot",
"accountName": "Snapshot Alpha",
"accountEmail": "alpha@example.invalid",
"authenticationKind": "oauth",
"imageInputTokens": 31,
"imageOutputTokens": 9,
"websocketPool": {"kind": "reuse"},
"imageGenerationRequested": true,
"imageGenerationSucceeded": true,
"requestedModel": "grok-4.5",
"upstreamModel": "grok-4.5",
"clientIp": "127.0.0.1",
"userAgent": "usage-metadata-test",
"reasoningEffort": "max",
"reasoningPreset": "ultra",
"requestKind": "review",
"subagentKind": "worker",
"compact": true,
"transport": "http_sse",
"transportDecisionWaitMs": 7,
"wsConnectMs": 11,
"firstReasoningMs": 19,
"firstTextMs": 23,
"firstTokenMs": 19,
"effectiveModel": "grok-4.5",
"requestSummary": {"inputItemsCount": 2},
})
);
}
@@ -62,6 +62,123 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
assert_eq!(request.validate().unwrap_err().field(), "rotationStrategy");
}
#[test]
fn settings_response_should_cover_the_full_runtime_settings_contract() {
use std::collections::BTreeMap;
use chrono::{TimeZone, Utc};
use gateway_admin::model::Revision;
use gateway_admin::model::settings::RuntimeSettings;
use gateway_api::admin::settings::RuntimeSettingsView;
use gateway_core::engine::credential::RotationStrategy;
use gateway_core::routing::{PublicModelId, UpstreamModelId};
let settings = RuntimeSettings {
config_revision: Revision::new(7).expect("revision"),
model_mappings: BTreeMap::from_iter([
(
PublicModelId::new("gpt-5.4").expect("public model"),
UpstreamModelId::new("gpt-5.5").expect("upstream model"),
),
(
PublicModelId::new("grok-latest").expect("public model"),
UpstreamModelId::new("grok-4.5").expect("upstream model"),
),
]),
refresh_margin_seconds: 1800,
refresh_concurrency: 4,
max_concurrent_per_account: 5,
request_interval_ms: 25,
rotation_strategy: RotationStrategy::RoundRobin,
usage_retention_days: 32,
ops_event_retention_days: 31,
audit_retention_days: 91,
updated_at: Utc
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
.single()
.expect("timestamp"),
};
let value = serde_json::to_value(RuntimeSettingsView::from(settings)).expect("serialize view");
assert_eq!(
value,
json!({
"modelMappings": {
"gpt-5.4": "gpt-5.5",
"grok-latest": "grok-4.5"
},
"refreshMarginSeconds": 1800,
"refreshConcurrency": 4,
"maxConcurrentPerAccount": 5,
"requestIntervalMs": 25,
"rotationStrategy": "round_robin",
"usageRetentionDays": 32,
"opsEventRetentionDays": 31,
"auditRetentionDays": 91,
"updatedAt": "2026-08-02T10:30:00Z"
})
);
}
#[test]
fn settings_request_and_response_fields_should_stay_in_lockstep() {
use std::collections::{BTreeMap, BTreeSet};
use gateway_admin::model::Revision;
use gateway_admin::model::settings::RuntimeSettings;
use gateway_api::admin::settings::RuntimeSettingsView;
use gateway_core::engine::credential::RotationStrategy;
use gateway_core::routing::{PublicModelId, UpstreamModelId};
let request: UpdateRuntimeSettingsRequest =
serde_json::from_value(update_body()).expect("decode settings");
request.validate().expect("fixture settings must validate");
let request_fields: BTreeSet<String> = update_body()
.as_object()
.expect("request body object")
.keys()
.cloned()
.collect();
let settings = RuntimeSettings {
config_revision: Revision::new(7).expect("revision"),
model_mappings: request
.model_mappings
.iter()
.map(|(public, upstream)| {
Ok((
PublicModelId::new(public.clone())?,
UpstreamModelId::new(upstream.clone())?,
))
})
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
.expect("valid model mappings"),
refresh_margin_seconds: request.refresh_margin_seconds,
refresh_concurrency: u32::try_from(request.refresh_concurrency).expect("u32"),
max_concurrent_per_account: u32::try_from(request.max_concurrent_per_account).expect("u32"),
request_interval_ms: request.request_interval_ms,
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
.expect("fixture rotation strategy"),
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
updated_at: chrono::Utc::now(),
};
let response_fields: BTreeSet<String> =
serde_json::to_value(RuntimeSettingsView::from(settings))
.expect("serialize view")
.as_object()
.expect("view object")
.keys()
.cloned()
.collect();
let mut expected_fields = request_fields;
expected_fields.insert("updatedAt".to_owned());
assert_eq!(response_fields, expected_fields);
}
#[test]
fn settings_request_should_reject_unknown_revision_field() {
let mut body = update_body();
@@ -35,9 +35,15 @@ fn source_tree_should_match_frozen_machine_manifest() {
let mut expected = vec![
"src/admin/accounts.rs",
"src/admin/auth.rs",
"src/admin/backups.rs",
"src/admin/client_keys.rs",
"src/admin/mod.rs",
"src/admin/observability.rs",
"src/admin/observability/mod.rs",
"src/admin/observability/presenter.rs",
"src/admin/observability/query.rs",
"src/admin/observability/routes.rs",
"src/admin/observability/wire.rs",
"src/admin/presenter.rs",
"src/admin/settings.rs",
"src/admin/system.rs",
"src/admin/wire.rs",
@@ -71,7 +77,9 @@ fn test_tree_should_match_frozen_rust_mirror() {
"tests/admin/auth.rs",
"tests/admin/client_keys.rs",
"tests/admin/mod.rs",
"tests/admin/observability.rs",
"tests/admin/observability/mod.rs",
"tests/admin/observability/query.rs",
"tests/admin/observability/response.rs",
"tests/admin/settings.rs",
"tests/admin/system.rs",
"tests/admin/wire.rs",
@@ -58,6 +58,30 @@ fn no_available_provider_error_should_map_to_service_unavailable() {
);
}
#[test]
fn account_capacity_error_should_map_to_service_unavailable() {
assert_eq!(
gateway_error_contract(GatewayErrorKind::AccountCapacityUnavailable),
(
StatusCode::SERVICE_UNAVAILABLE,
"server_error",
"account_capacity_unavailable",
)
);
}
#[test]
fn provider_infrastructure_error_should_map_to_service_unavailable() {
assert_eq!(
gateway_error_contract(GatewayErrorKind::ProviderInfrastructureUnavailable),
(
StatusCode::SERVICE_UNAVAILABLE,
"server_error",
"provider_infrastructure_unavailable",
)
);
}
#[test]
fn rate_limited_error_should_map_to_openai_retryable_status() {
assert_eq!(
@@ -122,29 +146,32 @@ fn engine_provider_error_should_preserve_retry_classification() {
#[test]
fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
let error = EngineError::Provider(ProviderError::new(
ProviderErrorKind::Unavailable,
ProviderErrorKind::AccountCapacityUnavailable,
UpstreamSendState::NotSent,
));
assert_eq!(
gateway_error_from_engine(&error),
GatewayError::new(
GatewayErrorKind::NoAvailableProvider,
"no upstream provider is currently available for this request"
GatewayErrorKind::AccountCapacityUnavailable,
"all eligible upstream accounts are temporarily busy"
)
);
}
#[test]
fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
fn no_eligible_provider_account_should_preserve_safe_provider_detail() {
let detail = ClientVisibleUpstreamError::new(
"no account is eligible for the requested model",
Some("no_eligible_account".to_owned()),
Some("account_unavailable_error".to_owned()),
);
let error = EngineError::Provider(
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
.with_client_visible_upstream_error(detail),
ProviderError::new(
ProviderErrorKind::NoEligibleAccount,
UpstreamSendState::NotSent,
)
.with_client_visible_upstream_error(detail),
);
let gateway = gateway_error_from_engine(&error);
@@ -164,6 +191,22 @@ fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
);
}
#[test]
fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
let error = EngineError::Provider(ProviderError::new(
ProviderErrorKind::ProviderInfrastructureUnavailable,
UpstreamSendState::NotSent,
));
assert_eq!(
gateway_error_from_engine(&error),
GatewayError::new(
GatewayErrorKind::ProviderInfrastructureUnavailable,
"provider account infrastructure is temporarily unavailable"
)
);
}
#[test]
fn no_eligible_provider_account_should_map_to_service_unavailable() {
let error = EngineError::Provider(ProviderError::new(
@@ -181,16 +224,18 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
}
#[test]
fn sent_provider_unavailability_should_remain_bad_gateway() {
let error = EngineError::Provider(ProviderError::new(
ProviderErrorKind::Unavailable,
UpstreamSendState::Sent,
));
fn provider_unavailability_should_remain_bad_gateway_regardless_of_send_state() {
for send_state in [UpstreamSendState::NotSent, UpstreamSendState::Sent] {
let error = EngineError::Provider(ProviderError::new(
ProviderErrorKind::Unavailable,
send_state,
));
assert_eq!(
gateway_error_from_engine(&error).kind(),
GatewayErrorKind::UpstreamUnavailable
);
assert_eq!(
gateway_error_from_engine(&error).kind(),
GatewayErrorKind::UpstreamUnavailable
);
}
}
#[test]
@@ -533,9 +533,8 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
operation_kind: OperationKind::Generate,
input: Some(json!("hello")),
client_metadata: None,
protocol_context: Some(json!({
"opaque_request_headers": [["user-agent", "IENvZGV4LUNMSS8xLjAg"]]
})),
// 客户端 User-Agent 仅用于本地展示,不再透传给上游指纹上下文。
protocol_context: None,
prompt_cache_key: None,
previous_response_id: None,
}
@@ -288,6 +288,15 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
"x-codex-installation-id",
HeaderValue::from_static("client-installation"),
);
headers.insert(
"x-oai-attestation",
HeaderValue::from_static("client-attestation"),
);
headers.insert("x-oai-is", HeaderValue::from_static("client-is"));
headers.insert(
"x-oai-is-update",
HeaderValue::from_static("client-is-update"),
);
let decoded =
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
@@ -321,8 +330,6 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
for preserved in [
"accept",
"content-type",
"user-agent",
"originator",
"x-openai-internal-codex-residency",
"x-codex-turn-state",
] {
@@ -339,6 +346,12 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
"chatgpt-account-id",
"chatgpt-project-id",
"x-codex-installation-id",
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
"user-agent",
"originator",
"x-oai-attestation",
"x-oai-is",
"x-oai-is-update",
] {
assert!(values(excluded).is_empty(), "unexpected {excluded}");
}
@@ -39,6 +39,30 @@ impl Decimal {
.filter(|value| *value <= MAX_SCALED_DECIMAL)
.map(Self)
}
/// 除以非零整数,保留最多十位小数。
#[must_use]
pub fn checked_div_u64(self, divisor: u64) -> Option<Self> {
let divisor = u128::from(divisor);
(divisor != 0)
.then(|| self.0.checked_div(divisor))
.flatten()
.and_then(|value| Self::from_scaled(value).ok())
}
/// 去尾零的 canonical 字符串,用于 wire 序列化。
#[must_use]
pub fn canonical(self) -> String {
let integer = self.0 / DECIMAL_SCALE;
let fraction = self.0 % DECIMAL_SCALE;
if fraction == 0 {
integer.to_string()
} else {
let fraction = format!("{fraction:010}");
let trimmed = fraction.trim_end_matches('0');
format!("{integer}.{trimmed}")
}
}
}
impl FromStr for Decimal {
@@ -190,9 +190,26 @@ impl fmt::Debug for ContinuationBinding {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
#[error("native continuation store is unavailable")]
pub struct NativeContinuationStoreError;
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error("native continuation store is unavailable: {detail}")]
pub struct NativeContinuationStoreError {
detail: String,
}
impl NativeContinuationStoreError {
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
#[must_use]
pub fn new(detail: impl Into<String>) -> Self {
Self {
detail: detail.into(),
}
}
#[must_use]
pub fn detail(&self) -> &str {
&self.detail
}
}
/// 可丢失的 previous-response 亲和存储端口。
///
@@ -165,8 +165,8 @@ where
timings: ModelRequestTimings::default(),
client_response_id: None,
upstream_response_id: None,
last_account_exhaustion: None,
last_account_exhaustion_events: Vec::new(),
last_retryable_failure: None,
last_retryable_failure_events: Vec::new(),
provider_attempt_outcomes: Vec::new(),
pending_terminal_failure: None,
};
@@ -245,9 +245,10 @@ pub struct ResponseExecutionSession<S: ?Sized> {
timings: ModelRequestTimings,
client_response_id: Option<String>,
upstream_response_id: Option<String>,
last_account_exhaustion: Option<ProviderError>,
/// 最近一次账号耗尽在重试时被丢弃的原始失败批次;只用于后续空选路终态。
last_account_exhaustion_events: Vec<ProviderEvent>,
/// 最近一次为无感恢复而被丢弃的原始上游错误;只在后续空选路时成为终态。
last_retryable_failure: Option<ProviderError>,
/// 与 `last_retryable_failure` 同属一个 attempt 的原始失败批次。
last_retryable_failure_events: Vec<ProviderEvent>,
provider_attempt_outcomes: Vec<ProviderAttemptOutcome>,
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误。
pending_terminal_failure: Option<PendingTerminalFailure>,
@@ -659,25 +660,30 @@ where
ProviderBoundary::Result(result) => match *result {
Ok(stream) => stream,
Err(error) => {
if error.kind() == ProviderErrorKind::NoEligibleAccount
&& let Some(exhaustion) = self.last_account_exhaustion.take()
if matches!(
error.kind(),
ProviderErrorKind::AccountCapacityUnavailable
| ProviderErrorKind::NoEligibleAccount
) && let Some(last_failure) = self.last_retryable_failure.take()
{
let send_state = self.current_send_state();
let mut events = std::mem::take(&mut self.last_account_exhaustion_events);
let mut events = std::mem::take(&mut self.last_retryable_failure_events);
if !events.is_empty() {
self.observe_atomic_terminal_events(&mut events);
return Ok(Some(PullOutcome::TerminalFailure {
events,
error: exhaustion,
error: last_failure,
send_state,
}));
}
self.finish_provider_error(&exhaustion).await?;
return Err(provider_engine_error(exhaustion));
self.finish_provider_error(&last_failure).await?;
return Err(provider_engine_error(last_failure));
}
if !(matches!(
error.kind(),
ProviderErrorKind::Unavailable | ProviderErrorKind::NoEligibleAccount
ProviderErrorKind::AccountCapacityUnavailable
| ProviderErrorKind::NoEligibleAccount
| ProviderErrorKind::ProviderInfrastructureUnavailable
) && error.send_state() == UpstreamSendState::NotSent)
{
self.record_provider_failure(candidate.provider().clone(), error.kind());
@@ -909,14 +915,10 @@ where
&mut self,
mut error: ProviderError,
) -> Result<StreamErrorOutcome, EngineError> {
// 原始 wire 只能活在本次决策栈内;后续 clone、attempt 记录与终态对象均只
// 接触已剥离的稳定错误字段。
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
// 均只接触已剥离的稳定错误字段。
let mut atomic_client_events = error.take_atomic_client_events();
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
let account_exhausted = matches!(
error.kind(),
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted
);
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
// attempt_send_state 是本 attempt 自身的发送事实,驱动重试门;
// 持久化与终态用请求级水位,二者不可混用(水位会把早先 attempt 的
@@ -976,8 +978,8 @@ where
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
if retryable {
// 普通 clone 只保留稳定事实;原始失败响应只能随最后一次客户端可见
// 的账号耗尽错误继续,不能进入中间失败记录。
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
// 所有权保留到下一次 attempt 成功,或最终空选路时返回客户端。
let persistence_error = self.request_persisted.then(|| error.clone());
if same_account_retry {
if let Some(account) = current.metadata.provider_account_id() {
@@ -992,10 +994,8 @@ where
{
self.excluded_accounts.insert(account.clone());
}
if account_exhausted {
self.last_account_exhaustion_events = atomic_client_events;
self.last_account_exhaustion = Some(error);
}
self.last_retryable_failure_events = atomic_client_events;
self.last_retryable_failure = Some(error);
if let Some(error) = persistence_error {
best_effort_store_write(
"record_intermediate_failure",
@@ -188,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData {
}
/// 数据库中固定的账号运行状态。
///
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AccountAvailability {
Unknown,
Ready,
Cooldown,
QuotaExhausted,
Expired,
Banned,
@@ -206,7 +208,6 @@ impl AccountAvailability {
match self {
Self::Unknown => "unknown",
Self::Ready => "ready",
Self::Cooldown => "cooldown",
Self::QuotaExhausted => "quota_exhausted",
Self::Expired => "expired",
Self::Banned => "banned",
@@ -220,7 +221,6 @@ impl AccountAvailability {
match value {
"unknown" => Some(Self::Unknown),
"ready" => Some(Self::Ready),
"cooldown" => Some(Self::Cooldown),
"quota_exhausted" => Some(Self::QuotaExhausted),
"expired" => Some(Self::Expired),
"banned" => Some(Self::Banned),
@@ -244,7 +244,6 @@ pub struct ProviderAccount {
revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
cooldown_until: Option<SystemTime>,
access_token_expires_at: Option<SystemTime>,
next_refresh_at: Option<SystemTime>,
has_refresh_token: bool,
@@ -274,7 +273,6 @@ impl ProviderAccount {
revision,
enabled: true,
availability: AccountAvailability::Unknown,
cooldown_until: None,
access_token_expires_at,
next_refresh_at: None,
has_refresh_token: false,
@@ -299,11 +297,9 @@ impl ProviderAccount {
mut self,
enabled: bool,
availability: AccountAvailability,
cooldown_until: Option<SystemTime>,
) -> Self {
self.enabled = enabled;
self.availability = availability;
self.cooldown_until = cooldown_until;
self
}
@@ -373,11 +369,6 @@ impl ProviderAccount {
self.enabled
}
#[must_use]
pub const fn cooldown_until(&self) -> Option<SystemTime> {
self.cooldown_until
}
#[must_use]
pub const fn access_token_expires_at(&self) -> Option<SystemTime> {
self.access_token_expires_at
@@ -398,7 +389,6 @@ impl ProviderAccount {
pub fn is_schedulable(&self, now: SystemTime) -> bool {
let available = match self.availability {
AccountAvailability::Ready => true,
AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now),
AccountAvailability::Unknown
| AccountAvailability::QuotaExhausted
| AccountAvailability::Expired
@@ -588,6 +578,8 @@ pub struct QuotaObservation {
pub expected_revision: CredentialRevision,
pub quota: Option<OpaqueProviderData>,
pub observed_at: Option<SystemTime>,
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
pub limit_reached: Option<bool>,
}
impl fmt::Debug for QuotaObservation {
@@ -614,8 +606,6 @@ pub struct AccountStateChange {
pub account_id: ProviderAccountId,
pub expected_revision: CredentialRevision,
pub availability: AccountAvailability,
pub reason: Option<String>,
pub cooldown_until: Option<SystemTime>,
pub observed_at: SystemTime,
}
@@ -698,6 +688,17 @@ impl RotationStrategy {
Self::Sticky => "sticky",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"smart" => Some(Self::Smart),
"quota_reset_priority" => Some(Self::QuotaResetPriority),
"round_robin" => Some(Self::RoundRobin),
"sticky" => Some(Self::Sticky),
_ => None,
}
}
}
/// 从 `runtime_settings` 冻结到一次请求计划的账号调度策略。
@@ -745,6 +746,7 @@ pub struct AccountRuntimeSignals {
pub last_started_at: Option<SystemTime>,
pub quota_reset_at: Option<SystemTime>,
pub quota_remaining_rank: Option<u64>,
pub quota_limit_reached: bool,
pub failure_rate_basis_points: Option<u16>,
pub first_output_latency_ms: Option<u64>,
}
@@ -892,6 +894,7 @@ impl AccountRuntimeSignals {
if let Some(quota) = quota {
self.quota_reset_at = quota.reset_at;
self.quota_remaining_rank = quota.remaining_rank;
self.quota_limit_reached = quota.limit_reached;
}
self
}
@@ -913,14 +916,20 @@ impl AccountRuntimeSignals {
pub struct AccountQuotaSignals {
reset_at: Option<SystemTime>,
remaining_rank: Option<u64>,
limit_reached: bool,
}
impl AccountQuotaSignals {
#[must_use]
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
pub const fn new(
reset_at: Option<SystemTime>,
remaining_rank: Option<u64>,
limit_reached: bool,
) -> Self {
Self {
reset_at,
remaining_rank,
limit_reached,
}
}
@@ -933,6 +942,12 @@ impl AccountQuotaSignals {
pub const fn remaining_rank(self) -> Option<u64> {
self.remaining_rank
}
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
#[must_use]
pub const fn limit_reached(self) -> bool {
self.limit_reached
}
}
/// 账号持久事实与可重建运行信号的请求级组合。
@@ -3,6 +3,7 @@
use std::collections::BTreeSet;
use std::fmt;
use std::net::IpAddr;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
@@ -39,7 +40,7 @@ use crate::routing::{
};
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
const REDIS_COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
const COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClientTransport {
@@ -173,6 +174,22 @@ pub enum ProviderCircuitDecision {
#[error("provider circuit store is unavailable")]
pub struct ProviderCircuitError;
/// Provider circuit 的可重建协调策略;由 Core 拥有并交给 Store adapter 执行。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ProviderCircuitPolicy {
pub failure_threshold: NonZeroU32,
pub open_duration: Duration,
}
impl Default for ProviderCircuitPolicy {
fn default() -> Self {
Self {
failure_threshold: NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
open_duration: Duration::from_secs(30),
}
}
}
pub trait ProviderCircuitPort: Send + Sync {
fn decision<'a>(
&'a self,
@@ -254,10 +271,20 @@ impl DefaultExecutionService {
let continuation = match request.metadata.previous_response_id.as_ref() {
Some(previous) => {
let resolve = self.continuation.resolve(previous).fuse();
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(resolve, timeout);
let pin = select_biased! {
result = resolve => result.ok().flatten(),
result = resolve => match result {
Ok(pin) => pin,
Err(error) => {
tracing::debug!(
request_id = request_id.as_str(),
%error,
"Continuation affinity 查询失败,退化为外部续接"
);
None
}
},
_ = timeout => None,
};
match pin {
@@ -393,7 +420,7 @@ impl DefaultExecutionService {
provider_kind: &ProviderKind,
) -> Result<RoutingContext, GatewayError> {
let decision = self.circuits.decision(provider_kind).fuse();
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(decision, timeout);
let decision = select_biased! {
result = decision => Some(result),
@@ -933,17 +960,28 @@ async fn record_native_continuation(
continuation: &dyn NativeContinuationPort,
pin: NativeContinuationPin,
) {
let provider = pin.provider().as_str().to_owned();
let account = pin.account().as_str().to_owned();
let record = continuation.record(pin).fuse();
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(record, timeout);
select_biased! {
result = record => {
if let Err(error) = result {
tracing::warn!(%error, "Continuation affinity 写入失败,后续请求将退化为外部续接");
tracing::warn!(
provider = %provider,
account = %account,
%error,
"Continuation affinity 写入失败,后续请求将退化为外部续接"
);
}
},
_ = timeout => {
tracing::warn!("Continuation affinity 后台写入超时,已丢弃本次亲和记录");
tracing::warn!(
provider = %provider,
account = %account,
"Continuation affinity 后台写入超时,已丢弃本次亲和记录"
);
},
}
}
+20
View File
@@ -129,8 +129,12 @@ pub enum ProviderErrorKind {
RateLimited,
/// Credential 配额耗尽。
QuotaExhausted,
/// 仍有符合条件的账号,但它们暂时没有可调度容量。
AccountCapacityUnavailable,
/// Provider 已确认当前请求无法选出可用账号。
NoEligibleAccount,
/// Provider 的账号存储、租约协调或本地凭据数据不可用。
ProviderInfrastructureUnavailable,
/// 请求超时。
Timeout,
/// 网络或 transport 失败。
@@ -156,7 +160,9 @@ impl ProviderErrorKind {
Self::PermissionDenied => "permission_denied",
Self::RateLimited => "rate_limited",
Self::QuotaExhausted => "quota_exhausted",
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
Self::NoEligibleAccount => "no_eligible_account",
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
Self::Timeout => "timeout",
Self::Transport => "transport",
Self::Protocol => "protocol",
@@ -711,6 +717,10 @@ pub enum GatewayErrorKind {
ModelNotFound,
/// 当前没有可用 target。
NoAvailableProvider,
/// 符合条件的上游账号暂时没有调度容量。
AccountCapacityUnavailable,
/// Provider 的本地账号基础设施不可用。
ProviderInfrastructureUnavailable,
/// 上游限流。
RateLimited,
/// 上游暂不可用。
@@ -734,6 +744,8 @@ impl GatewayErrorKind {
Self::PolicyDenied => "policy_denied",
Self::ModelNotFound => "model_not_found",
Self::NoAvailableProvider => "no_available_provider",
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
Self::RateLimited => "rate_limited",
Self::UpstreamUnavailable => "upstream_unavailable",
Self::Timeout => "timeout",
@@ -782,10 +794,18 @@ impl GatewayError {
GatewayErrorKind::RateLimited,
"upstream capacity is temporarily unavailable",
),
ProviderErrorKind::AccountCapacityUnavailable => Self::new(
GatewayErrorKind::AccountCapacityUnavailable,
"all eligible upstream accounts are temporarily busy",
),
ProviderErrorKind::NoEligibleAccount => Self::new(
GatewayErrorKind::NoAvailableProvider,
"no upstream provider is currently available for this request",
),
ProviderErrorKind::ProviderInfrastructureUnavailable => Self::new(
GatewayErrorKind::ProviderInfrastructureUnavailable,
"provider account infrastructure is temporarily unavailable",
),
ProviderErrorKind::Timeout => {
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
}
@@ -635,6 +635,13 @@ pub trait ProviderCooldownPort: Send + Sync {
scope: &'a ProviderCooldownScope,
through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
/// 删除账号时清除该账号全部 account/model scope cooldown key。
/// 生命周期与 credential revision 无关;不要求调用方持有 revision。
fn clear_all<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -244,13 +244,8 @@ async fn compile_runtime_snapshot(
}
let model_mappings = facts.settings.model_mappings;
let rotation_strategy = match facts.settings.rotation_strategy.as_str() {
"smart" => RotationStrategy::Smart,
"quota_reset_priority" => RotationStrategy::QuotaResetPriority,
"round_robin" => RotationStrategy::RoundRobin,
"sticky" => RotationStrategy::Sticky,
_ => return Err(RuntimeSnapshotCompileError::InvalidData),
};
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
let selection_policy = AccountSelectionPolicy::new(
rotation_strategy,
NonZeroU32::new(facts.settings.max_concurrent_per_account)
+4 -1
View File
@@ -21,10 +21,11 @@ pub enum WorkerKind {
StaleModelRequestRecovery,
Retention,
OpsFlush,
Backup,
}
impl WorkerKind {
pub const ALL: [Self; 7] = [
pub const ALL: [Self; 8] = [
Self::OAuthRefresh,
Self::QuotaCatalogHealth,
Self::RuntimeSnapshotReconciliation,
@@ -32,6 +33,7 @@ impl WorkerKind {
Self::StaleModelRequestRecovery,
Self::Retention,
Self::OpsFlush,
Self::Backup,
];
#[must_use]
@@ -44,6 +46,7 @@ impl WorkerKind {
Self::StaleModelRequestRecovery => "stale_model_request_recovery",
Self::Retention => "retention",
Self::OpsFlush => "ops_flush",
Self::Backup => "backup",
}
}
}
@@ -12,6 +12,22 @@ fn decimal_should_render_with_database_scale() {
assert_eq!(decimal.to_string(), "12.3400000000");
}
#[test]
fn decimal_canonical_should_strip_trailing_zeroes() {
assert_eq!(Decimal::from_str("12.34").unwrap().canonical(), "12.34");
assert_eq!(Decimal::from_str("12").unwrap().canonical(), "12");
assert_eq!(Decimal::from_str("0").unwrap().canonical(), "0");
assert_eq!(Decimal::from_str("1.0000000000").unwrap().canonical(), "1");
}
#[test]
fn decimal_div_u64_should_keep_up_to_ten_fraction_digits() {
let decimal = Decimal::from_str("10").unwrap();
let third = decimal.checked_div_u64(3).unwrap();
assert_eq!(third.canonical(), "3.3333333333");
assert_eq!(decimal.checked_div_u64(0), None);
}
#[test]
fn provider_usd_ticks_should_map_exactly_to_decimal_scale() {
let estimate = ProviderReportedCost::from_usd_ticks(12_345_678_901)
@@ -29,8 +29,9 @@ use gateway_core::engine::{
ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
};
use gateway_core::error::{
ClientVisibleUpstreamResponse, ContinuationFailure, GatewayErrorKind, OpaqueUpstreamValue,
ProviderError, ProviderErrorKind, StoreError, StoreErrorKind,
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
GatewayErrorKind, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, StoreError,
StoreErrorKind,
};
use gateway_core::event::{
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
@@ -2554,6 +2555,128 @@ fn pre_delivery_transport_failure_rotates_account_for_non_idempotent_generation(
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
}
#[test]
fn final_capacity_exhaustion_returns_the_last_retryable_upstream_failure() {
let operation = operation(RetrySafety::NonIdempotent);
let route_plan = plan(&operation);
let raw_body = Bytes::from_static(b"{\"error\":\"last upstream body\"}\x00");
let upstream_error =
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
.with_pre_delivery_retry()
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"message too big",
Some("1009".to_owned()),
Some("websocket_close_error".to_owned()),
))
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
502,
Some(b"application/json".to_vec()),
raw_body.clone(),
));
let (coordinator, store, provider) = coordinator(vec![
Script::Stream {
account_id: "acct_first",
items: vec![Err(upstream_error)],
},
Script::Error(ProviderError::new(
ProviderErrorKind::AccountCapacityUnavailable,
UpstreamSendState::NotSent,
)),
]);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
None,
CancellationToken::new(),
))
.expect("start execution");
let error = block_on(session.collect_uncommitted())
.expect_err("empty selection must return the retained upstream failure");
let EngineError::Provider(error) = error else {
panic!("expected retained provider error");
};
assert_eq!(error.kind(), ProviderErrorKind::Transport);
let detail = error
.client_visible_upstream_error()
.expect("structured upstream detail");
assert_eq!(detail.message(), "message too big");
assert_eq!(detail.code(), Some("1009"));
assert_eq!(detail.error_type(), Some("websocket_close_error"));
let response = error
.client_visible_upstream_response()
.expect("original upstream response");
assert_eq!(response.status(), 502);
assert_eq!(response.body(), &raw_body);
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
let state = store.state.lock().expect("store lock");
assert_eq!(state.intermediate_failures, 1);
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
assert_eq!(
state.finalizations[0].send_state,
UpstreamSendState::Ambiguous
);
}
#[test]
fn provider_infrastructure_failure_does_not_restore_the_last_upstream_failure() {
let operation = operation(RetrySafety::NonIdempotent);
let route_plan = plan(&operation);
let upstream_error =
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
.with_pre_delivery_retry()
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream marker",
Some("upstream_marker".to_owned()),
Some("upstream_error".to_owned()),
))
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
502,
Some(b"application/json".to_vec()),
Bytes::from_static(b"{\"error\":\"upstream marker\"}"),
));
let (coordinator, store, provider) = coordinator(vec![
Script::Stream {
account_id: "acct_first",
items: vec![Err(upstream_error)],
},
Script::Error(ProviderError::new(
ProviderErrorKind::ProviderInfrastructureUnavailable,
UpstreamSendState::NotSent,
)),
]);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
None,
CancellationToken::new(),
))
.expect("start execution");
let error = block_on(session.collect_uncommitted())
.expect_err("provider infrastructure failure must remain terminal");
let EngineError::Provider(error) = error else {
panic!("expected provider error");
};
assert_eq!(
error.kind(),
ProviderErrorKind::ProviderInfrastructureUnavailable
);
assert!(error.client_visible_upstream_error().is_none());
assert!(error.client_visible_upstream_response().is_none());
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
let state = store.state.lock().expect("store lock");
assert_eq!(state.intermediate_failures, 1);
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
}
#[test]
fn pre_delivery_retry_marker_is_ignored_after_downstream_commit() {
let operation = operation(RetrySafety::NonIdempotent);
@@ -24,7 +24,7 @@ fn account(id: &str) -> ProviderAccount {
CredentialRevision::new(1).expect("valid revision"),
Some(SystemTime::now() + Duration::from_secs(3600)),
)
.with_runtime_state(true, AccountAvailability::Ready, None)
.with_runtime_state(true, AccountAvailability::Ready)
}
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
@@ -35,6 +35,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: remaining,
quota_limit_reached: false,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -56,6 +57,19 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
}
}
#[test]
fn rotation_strategy_parse_should_round_trip_stable_wire_values() {
for strategy in [
RotationStrategy::Smart,
RotationStrategy::QuotaResetPriority,
RotationStrategy::RoundRobin,
RotationStrategy::Sticky,
] {
assert_eq!(RotationStrategy::parse(strategy.as_str()), Some(strategy));
}
assert_eq!(RotationStrategy::parse("unknown"), None);
}
#[test]
fn plaintext_credential_debug_should_redact_values() {
let mut object = Map::new();
@@ -97,31 +111,27 @@ fn availability_should_round_trip_all_database_values() {
#[test]
fn diagnostic_selection_bypasses_all_local_account_availability() {
let exhausted = AccountCandidate {
account: account("acct_exhausted").with_runtime_state(
true,
AccountAvailability::QuotaExhausted,
None,
),
account: account("acct_exhausted")
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
};
let disabled = AccountCandidate {
account: account("acct_disabled").with_runtime_state(
false,
AccountAvailability::QuotaExhausted,
None,
),
account: account("acct_disabled")
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -145,33 +155,11 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
#[test]
fn disabled_account_should_not_be_schedulable() {
let account =
account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready, None);
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
assert!(!account.is_schedulable(SystemTime::now()));
}
#[test]
fn cooldown_account_should_become_schedulable_at_its_deadline() {
let now = SystemTime::now();
let account =
account("acct_cooldown").with_runtime_state(true, AccountAvailability::Cooldown, Some(now));
assert!(account.is_schedulable(now));
}
#[test]
fn cooldown_account_should_remain_isolated_before_its_deadline() {
let now = SystemTime::now();
let account = account("acct_cooldown").with_runtime_state(
true,
AccountAvailability::Cooldown,
Some(now + Duration::from_secs(1)),
);
assert!(!account.is_schedulable(now));
}
#[test]
fn smart_selector_should_prefer_lower_inflight_count() {
let candidates = vec![
@@ -298,10 +286,15 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
last_started_at: Some(last_started_at),
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
failure_rate_basis_points: None,
first_output_latency_ms: None,
}
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
.with_provider_quota(Some(AccountQuotaSignals::new(
Some(reset_at),
Some(75),
false,
)));
assert_eq!(
(
@@ -616,3 +609,48 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
.is_err()
);
}
#[test]
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
// availability 终态(Banned/Expired/Invalid)始终排除,
// 不受 quota 信号或 skip_exhausted 放宽影响。
let mut banned = candidate("acct_banned", 0, Some(100));
banned.account = banned
.account
.with_runtime_state(true, AccountAvailability::Banned);
let mut expired = candidate("acct_expired", 0, Some(100));
expired.account = expired
.account
.with_runtime_state(true, AccountAvailability::Expired);
let mut invalid = candidate("acct_invalid", 0, Some(100));
invalid.account = invalid
.account
.with_runtime_state(true, AccountAvailability::Invalid);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![banned, expired, invalid, healthy];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("healthy candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
#[test]
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
exhausted.account = exhausted
.account
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![exhausted, healthy];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("healthy candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
@@ -101,6 +101,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: Some(7),
quota_limit_reached: false,
failure_rate_basis_points: Some(125),
first_output_latency_ms: Some(250),
},
@@ -17,13 +17,14 @@ use gateway_core::task::{
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
use tokio::sync::Notify;
const ACTIVE_KINDS: [WorkerKind; 6] = [
const ACTIVE_KINDS: [WorkerKind; 7] = [
WorkerKind::OAuthRefresh,
WorkerKind::QuotaCatalogHealth,
WorkerKind::RuntimeSnapshotReconciliation,
WorkerKind::RuntimeChangeSubscription,
WorkerKind::StaleModelRequestRecovery,
WorkerKind::Retention,
WorkerKind::Backup,
];
#[derive(Clone)]
+9 -2
View File
@@ -13,24 +13,31 @@ path = "tests/main.rs"
[dependencies]
async-trait = "0.1.89"
aws-credential-types = "1.2"
aws-sdk-s3 = { version = "1.140", default-features = false, features = ["rt-tokio", "default-https-client"] }
aws-smithy-runtime-api = "1.14"
aws-smithy-types = "1.6"
chrono = "0.4.45"
fs2 = "0.4"
futures = "0.3.32"
gateway-admin = { path = "../gateway-admin" }
gateway-core = { path = "../gateway-core" }
hex = "0.4.3"
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
secrecy = "0.10"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
sha2 = "0.11.0"
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json", "macros", "migrate"] }
thiserror = "2.0.18"
tokio = { version = "1.52.3", default-features = false, features = ["rt", "time"] }
tokio = { version = "1.52", default-features = false, features = ["fs", "io-std", "io-util", "macros", "process", "rt", "sync", "time"] }
tracing = "0.1.44"
url = "2.5.8"
uuid = { version = "1.23.3", features = ["v4", "v7"] }
[dev-dependencies]
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread"] }
tempfile = "3.23"
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "process", "time"] }
[lints]
workspace = true
@@ -0,0 +1,296 @@
//! Admin 认证与设置 adapter。
use super::*;
pub(crate) struct AdminAuthStoreAdapter {
pub(crate) security: postgres::PgAdminSecurityAuditRepository,
pub(crate) settings: postgres::PgRuntimeSettingsRepository,
pub(crate) state: redis::RedisAdminAuthStateRepository,
}
pub(crate) struct AdminSettingsStoreAdapter {
pub(crate) control_plane: postgres::PgControlPlaneRepository,
}
#[async_trait::async_trait]
impl SettingsStore for AdminSettingsStoreAdapter {
async fn load_runtime_settings(&self) -> AdminStoreResult<AdminRuntimeSettings> {
let snapshot = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
.await
.map_err(|error| admin_store_error("runtime settings", error))?;
admin_runtime_settings(snapshot.settings)
}
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
.await
.map(|snapshot| snapshot.settings.admin_api_key.is_some())
.map_err(|error| admin_store_error("admin API key", error))
}
async fn replace_runtime_settings(
&self,
command: ReplaceRuntimeSettings,
context: &MutationContext,
) -> AdminStoreResult<AdminRuntimeSettings> {
let current = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
.await
.map_err(|error| admin_store_error("runtime settings", error))?;
let replacement = postgres::ControlPlaneReplacement {
settings: postgres::RuntimeSettingsUpdate {
admin_api_key: current.settings.admin_api_key,
refresh_margin_seconds: command.refresh_margin_seconds,
refresh_concurrency: command.refresh_concurrency,
max_concurrent_per_account: command.max_concurrent_per_account,
request_interval_ms: command.request_interval_ms,
rotation_strategy: command.rotation_strategy.as_str().to_owned(),
model_mappings: store_model_mappings(command.model_mappings),
usage_retention_days: command.usage_retention_days,
ops_event_retention_days: command.ops_event_retention_days,
audit_retention_days: command.audit_retention_days,
},
audit: mutation_audit(
context,
"settings.replace",
"runtime_settings",
"1",
vec![
"model_mappings_json".to_owned(),
"refresh_margin_seconds".to_owned(),
"refresh_concurrency".to_owned(),
"max_concurrent_per_account".to_owned(),
"request_interval_ms".to_owned(),
"rotation_strategy".to_owned(),
"retention".to_owned(),
],
),
};
let snapshot = postgres::ControlPlaneRepository::replace_control_plane(
&self.control_plane,
replacement,
)
.await
.map_err(|error| admin_store_error("runtime settings", error))?;
admin_runtime_settings(snapshot.settings)
}
async fn replace_admin_api_key(
&self,
key: AdminApiKey,
context: &MutationContext,
) -> AdminStoreResult<AdminApiKeyMutation> {
self.replace_admin_api_key_value(Some(key.expose_for_auth().to_owned()), context)
.await
}
async fn delete_admin_api_key(
&self,
context: &MutationContext,
) -> AdminStoreResult<AdminApiKeyMutation> {
self.replace_admin_api_key_value(None, context).await
}
}
impl AdminSettingsStoreAdapter {
async fn replace_admin_api_key_value(
&self,
admin_api_key: Option<String>,
context: &MutationContext,
) -> AdminStoreResult<AdminApiKeyMutation> {
let exists = admin_api_key.is_some();
let revision = postgres::ControlPlaneRepository::replace_admin_api_key(
&self.control_plane,
admin_api_key,
mutation_audit(
context,
if exists {
"admin_api_key.replace"
} else {
"admin_api_key.delete"
},
"runtime_settings",
"1",
vec!["admin_api_key".to_owned()],
),
)
.await
.map_err(|error| admin_store_error("admin API key", error))?;
Ok(AdminApiKeyMutation {
config_revision: admin_revision(revision)?,
exists,
})
}
}
pub(crate) fn admin_runtime_settings(
settings: postgres::RuntimeSettings,
) -> AdminStoreResult<AdminRuntimeSettings> {
let rotation_strategy = AdminRotationStrategy::parse(settings.rotation_strategy.as_str())
.ok_or_else(|| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
"runtime settings",
"rotation strategy is invalid",
)
})?;
let model_mappings = settings
.model_mappings
.into_iter()
.map(|(public, upstream)| {
let public = gateway_core::routing::PublicModelId::new(public).map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
"runtime settings",
"public model mapping is invalid",
)
})?;
let upstream = gateway_core::routing::UpstreamModelId::new(upstream).map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
"runtime settings",
"upstream model mapping is invalid",
)
})?;
Ok((public, upstream))
})
.collect::<AdminStoreResult<ModelMappings>>()?;
Ok(AdminRuntimeSettings {
config_revision: admin_revision(settings.config_revision)?,
model_mappings,
refresh_margin_seconds: settings.refresh_margin_seconds,
refresh_concurrency: settings.refresh_concurrency,
max_concurrent_per_account: settings.max_concurrent_per_account,
request_interval_ms: settings.request_interval_ms,
rotation_strategy,
usage_retention_days: settings.usage_retention_days,
ops_event_retention_days: settings.ops_event_retention_days,
audit_retention_days: settings.audit_retention_days,
updated_at: settings.updated_at,
})
}
pub(crate) fn store_model_mappings(
mappings: ModelMappings,
) -> std::collections::BTreeMap<String, String> {
mappings
.into_iter()
.map(|(public, upstream)| (public.as_str().to_owned(), upstream.as_str().to_owned()))
.collect()
}
#[async_trait::async_trait]
impl AuthStore for AdminAuthStoreAdapter {
async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult<Option<String>> {
postgres::AdminSecurityAuditRepository::password_hash(&self.security, admin_user_id)
.await
.map_err(|error| admin_store_error("admin authentication", error))
}
async fn create_password_hash_if_absent(
&self,
admin_user_id: &str,
password_hash: &str,
) -> AdminStoreResult<bool> {
postgres::AdminSecurityAuditRepository::create_password_hash_if_absent(
&self.security,
admin_user_id,
password_hash,
)
.await
.map_err(|error| admin_store_error("admin authentication", error))
}
async fn load_admin_api_key(&self) -> AdminStoreResult<Option<AdminApiKey>> {
postgres::RuntimeSettingsRepository::load_runtime_settings(&self.settings)
.await
.map(|settings| settings.admin_api_key.map(AdminApiKey::new))
.map_err(|error| admin_store_error("admin API key", error))
}
async fn load_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>> {
redis::AdminAuthStateRepository::load_admin_session(&self.state, session_id)
.await
.map(|session| {
session.map(|record| AdminSession {
admin_user_id: record.admin_user_id,
expires_at: record.expires_at,
})
})
.map_err(|error| admin_store_error("admin session", error))
}
async fn store_session(
&self,
session_id: &str,
session: &AdminSession,
) -> AdminStoreResult<()> {
redis::AdminAuthStateRepository::store_admin_session(
&self.state,
session_id,
&redis::AdminSessionRecord {
admin_user_id: session.admin_user_id.clone(),
expires_at: session.expires_at,
},
)
.await
.map_err(|error| admin_store_error("admin session", error))
}
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>> {
redis::AdminAuthStateRepository::delete_admin_session(&self.state, session_id)
.await
.map(|session| {
session.map(|record| AdminSession {
admin_user_id: record.admin_user_id,
expires_at: record.expires_at,
})
})
.map_err(|error| admin_store_error("admin session", error))
}
async fn append_audit_event(&self, event: AdminAuditModel) -> AdminStoreResult<()> {
let config_revision = event
.config_revision
.map(|revision| i64::try_from(revision.get()))
.transpose()
.map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
"admin audit",
"config revision is outside the supported range",
)
})?;
let actor_kind = match event.actor_kind {
gateway_admin::model::auth::AuditActorKind::AdminSession => {
postgres::AdminAuditActorKind::AdminSession
}
gateway_admin::model::auth::AuditActorKind::AdminApiKey => {
postgres::AdminAuditActorKind::AdminApiKey
}
gateway_admin::model::auth::AuditActorKind::System => {
postgres::AdminAuditActorKind::System
}
gateway_admin::model::auth::AuditActorKind::Anonymous => {
postgres::AdminAuditActorKind::Anonymous
}
};
postgres::AdminSecurityAuditRepository::append_admin_audit_event(
&self.security,
postgres::AdminAuditEvent {
id: event.id,
actor_kind,
actor_admin_user_id: event.actor_admin_user_id,
actor_ref: event.actor_ref,
admin_request_id: event.request_id,
action: event.action,
entity_kind: event.entity_kind,
entity_ref: event.entity_ref,
config_revision,
changed_fields: event.changed_fields,
created_at: event.occurred_at,
},
)
.await
.map_err(|error| admin_store_error("admin audit", error))
}
}
@@ -0,0 +1,5 @@
//! S3 对象存储与 `pg_dump` 基础设施适配器。
pub mod pg_dump;
pub mod s3;
pub mod staging;
@@ -0,0 +1,205 @@
//! `pg_dump` 子进程与本地暂存适配器。
//!
//! 数据库密码只通过 `PGPASSWORD` 环境变量传给子进程,绝不进入命令行参数或日志。
//! 导出以有界内存流式写入暂存文件并计算 SHA-256;取消时终止子进程并清理部分文件。
use std::process::Stdio;
use std::sync::Arc;
use async_trait::async_trait;
use sha2::{Digest as _, Sha256};
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
use tokio::process::Command;
use gateway_admin::model::backup::{BackupError, code};
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpArtifact, DumpRequest, StagedArtifact};
use super::staging::StagingArea;
/// `pg_dump` 导出适配器。
pub struct PgDumpAdapter {
staging: Arc<StagingArea>,
/// 不含密码的 PostgreSQL URL,用于 `--dbname`。
database_url: String,
/// 数据库密码,仅通过 `PGPASSWORD` 注入。
database_password: String,
}
impl PgDumpAdapter {
/// 组合暂存区与数据库连接事实。
#[must_use]
pub fn new(
staging: Arc<StagingArea>,
database_url: impl Into<String>,
database_password: impl Into<String>,
) -> Self {
Self {
staging,
database_url: database_url.into(),
database_password: database_password.into(),
}
}
}
#[async_trait]
impl DatabaseDumpPort for PgDumpAdapter {
async fn dump(&self, request: DumpRequest) -> Result<DumpArtifact, BackupError> {
self.staging
.ensure_capacity()
.map_err(|_| staging_space_exhausted())?;
let partial = self.staging.partial_path(&request.backup_id);
let final_path = self.staging.final_path(&request.backup_id);
let mut child = Command::new("pg_dump")
.arg("--format=custom")
.arg("--no-owner")
.arg("--no-privileges")
.arg("--dbname")
.arg(&self.database_url)
.env("PGPASSWORD", &self.database_password)
.env("PGCONNECT_TIMEOUT", "10")
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(|_| pg_dump_failed("无法启动 pg_dump 进程"))?;
let stdout = child
.stdout
.take()
.ok_or_else(|| pg_dump_failed("无法读取 pg_dump 输出"))?;
let file = tokio::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.mode(0o600)
.open(&partial)
.await
.map_err(|_| pg_dump_failed("无法创建暂存文件"))?;
let mut writer = tokio::io::BufWriter::new(file);
let mut reader = tokio::io::BufReader::new(stdout);
let mut hasher = Sha256::new();
let mut size: u64 = 0;
let copy = async {
let mut buffer = vec![0_u8; 64 * 1024];
loop {
let n = reader
.read(&mut buffer)
.await
.map_err(|_| pg_dump_failed("读取 pg_dump 输出失败"))?;
if n == 0 {
break;
}
size = size.saturating_add(n as u64);
if size > self.staging.max_archive_bytes() {
return Err(staging_space_exhausted());
}
hasher.update(&buffer[..n]);
writer
.write_all(&buffer[..n])
.await
.map_err(|_| pg_dump_failed("写入暂存文件失败"))?;
}
writer
.flush()
.await
.map_err(|_| pg_dump_failed("刷新暂存文件失败"))?;
Ok(())
};
let copy_result = tokio::select! {
result = copy => result,
_ = request.cancellation.cancelled() => {
let _ = child.kill().await;
Err(cancelled())
}
};
if let Err(error) = copy_result {
let _ = child.kill().await;
let _ = child.wait().await;
self.staging.cleanup(&request.backup_id);
return Err(error);
}
let status = child
.wait()
.await
.map_err(|_| pg_dump_failed("等待 pg_dump 退出失败"))?;
if !status.success() {
self.staging.cleanup(&request.backup_id);
return Err(pg_dump_failed("pg_dump 非零退出"));
}
tokio::fs::rename(&partial, &final_path)
.await
.map_err(|_| pg_dump_failed("暂存归档改名失败"))?;
let sha256 = hex::encode(hasher.finalize());
Ok(DumpArtifact {
path: final_path,
size_bytes: size,
sha256,
})
}
async fn inspect_staging(
&self,
backup_id: &str,
) -> Result<Option<StagedArtifact>, BackupError> {
let path = self.staging.final_path(backup_id);
let metadata = match tokio::fs::metadata(&path).await {
Ok(metadata) => metadata,
Err(_) => return Ok(None),
};
if !metadata.is_file() {
return Ok(None);
}
let size_bytes = metadata.len();
let sha256 = hash_file(&path).await?;
Ok(Some(StagedArtifact {
path,
size_bytes,
sha256,
}))
}
async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError> {
self.staging.cleanup(backup_id);
Ok(())
}
}
/// 以有界内存读取文件并计算 SHA-256。
async fn hash_file(path: &std::path::Path) -> Result<String, BackupError> {
let mut file = tokio::fs::File::open(path)
.await
.map_err(|_| pg_dump_failed("打开暂存归档失败"))?;
let mut hasher = Sha256::new();
let mut buffer = vec![0_u8; 64 * 1024];
loop {
let n = file
.read(&mut buffer)
.await
.map_err(|_| pg_dump_failed("读取暂存归档失败"))?;
if n == 0 {
break;
}
hasher.update(&buffer[..n]);
}
Ok(hex::encode(hasher.finalize()))
}
fn pg_dump_failed(message: &'static str) -> BackupError {
BackupError::new(code::PG_DUMP_FAILED, message.to_owned())
}
fn staging_space_exhausted() -> BackupError {
BackupError::new(
code::STAGING_SPACE_EXHAUSTED,
"暂存磁盘空间不足或归档超过单任务上限".to_owned(),
)
}
fn cancelled() -> BackupError {
BackupError::new(code::CANCELLED, "备份导出已取消".to_owned())
}
@@ -0,0 +1,544 @@
//! S3 兼容对象存储适配器(含 Cloudflare R2)。
//!
//! 单个 `BackupObjectStorePort` 实现:方法接受已经校验的存储配置,内部按
//! `storage_revision` 缓存 SDK client。上传采用分片流式读取,内存占用有界;
//! 取消时主动 `AbortMultipartUpload`。
use std::collections::HashMap;
use std::path::Path;
use std::sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
};
use std::time::Duration;
use aws_credential_types::Credentials;
use aws_sdk_s3::config::{BehaviorVersion, Builder as S3ConfigBuilder, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::{Client as S3Client, presigning::PresigningConfig};
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
use aws_smithy_runtime_api::client::result::SdkError;
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
use aws_smithy_types::retry::RetryConfig;
use aws_smithy_types::timeout::TimeoutConfig;
use chrono::{DateTime, Utc};
use secrecy::ExposeSecret as _;
use tokio::io::{AsyncReadExt as _, AsyncSeekExt as _, SeekFrom};
use tokio::sync::Semaphore;
use uuid::Uuid;
use gateway_admin::model::backup::{
BackupError, BackupObjectMetadata, BackupStorageConfig, ConnectionTestResult,
ConnectionTestStage, code,
};
use gateway_admin::ports::backup::{BackupObjectStorePort, UploadObjectRequest};
/// 分片大小:16 MiB。
const PART_SIZE: usize = 16 * 1024 * 1024;
/// 并发上传分片数;最大额外内存约为 `PART_SIZE * 并发数`。
const CONCURRENCY: usize = 4;
/// SDK 单次请求超时。
const OPERATION_TIMEOUT: Duration = Duration::from_secs(300);
/// 探针对象的固定 metadata 值。
const PROBE_METADATA_KEY: &str = "probe";
const PROBE_METADATA_VALUE: &str = "backup-connection-test";
/// S3 兼容对象存储适配器。
pub struct S3ObjectStoreAdapter {
clients: Mutex<HashMap<u64, S3Client>>,
}
impl Default for S3ObjectStoreAdapter {
fn default() -> Self {
Self::new()
}
}
impl S3ObjectStoreAdapter {
/// 创建适配器;client 按 `storage_revision` 惰性构建并缓存。
#[must_use]
pub fn new() -> Self {
Self {
clients: Mutex::new(HashMap::new()),
}
}
fn client(&self, config: &BackupStorageConfig) -> S3Client {
if let Some(client) = self
.clients
.lock()
.ok()
.and_then(|guard| guard.get(&config.storage_revision).cloned())
{
return client;
}
let client = build_client(config);
if let Ok(mut guard) = self.clients.lock() {
guard.insert(config.storage_revision, client.clone());
}
client
}
}
fn build_client(config: &BackupStorageConfig) -> S3Client {
let retry = RetryConfig::standard().with_max_attempts(3);
let timeout = TimeoutConfig::builder()
.connect_timeout(Duration::from_secs(10))
.read_timeout(OPERATION_TIMEOUT)
.build();
let s3_config = S3ConfigBuilder::new()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(config.region.clone()))
.endpoint_url(&config.endpoint)
.force_path_style(config.force_path_style)
.credentials_provider(Credentials::new(
config.access_key_id.clone(),
config.secret_access_key.expose_secret().to_string(),
None,
None,
"static",
))
.retry_config(retry)
.timeout_config(timeout)
.build();
S3Client::from_conf(s3_config)
}
#[async_trait::async_trait]
impl BackupObjectStorePort for S3ObjectStoreAdapter {
async fn test_connection(
&self,
config: &BackupStorageConfig,
) -> Result<ConnectionTestResult, BackupError> {
let client = self.client(config);
let probe_key = format!(
"{}/.probe/{}",
config.prefix.trim_end_matches('/'),
Uuid::new_v4()
);
let body: Vec<u8> = (0_u8..64).collect();
if let Err(error) = client
.put_object()
.bucket(&config.bucket)
.key(&probe_key)
.body(ByteStream::from(body.clone()))
.metadata(PROBE_METADATA_KEY, PROBE_METADATA_VALUE)
.send()
.await
{
let _ = best_effort_delete(&client, config, &probe_key).await;
return Ok(failed(ConnectionTestStage::PutObject, &error));
}
match client
.head_object()
.bucket(&config.bucket)
.key(&probe_key)
.send()
.await
{
Ok(head) => {
let length_matches = head.content_length() == Some(body.len() as i64);
let metadata_matches = head
.metadata()
.and_then(|metadata| metadata.get(PROBE_METADATA_KEY))
.is_some_and(|value| value == PROBE_METADATA_VALUE);
if !length_matches || !metadata_matches {
let _ = best_effort_delete(&client, config, &probe_key).await;
return Ok(ConnectionTestResult {
ok: false,
stage: ConnectionTestStage::HeadObject.as_str(),
code: Some(code::UPSTREAM_FAILURE),
message: "探针对象校验不一致".to_owned(),
});
}
}
Err(error) => {
let _ = best_effort_delete(&client, config, &probe_key).await;
return Ok(failed(ConnectionTestStage::HeadObject, &error));
}
}
match client
.get_object()
.bucket(&config.bucket)
.key(&probe_key)
.send()
.await
{
Ok(get) => {
let received = get
.body
.collect()
.await
.map_err(|_| {
BackupError::new(code::UPSTREAM_FAILURE, "读取探针对象响应失败".to_owned())
})?
.into_bytes();
if received.as_ref() != body.as_slice() {
let _ = best_effort_delete(&client, config, &probe_key).await;
return Ok(ConnectionTestResult {
ok: false,
stage: ConnectionTestStage::GetObject.as_str(),
code: Some(code::UPSTREAM_FAILURE),
message: "探针对象内容不一致".to_owned(),
});
}
}
Err(error) => {
let _ = best_effort_delete(&client, config, &probe_key).await;
return Ok(failed(ConnectionTestStage::GetObject, &error));
}
}
let _ = best_effort_delete(&client, config, &probe_key).await;
Ok(ConnectionTestResult {
ok: true,
stage: ConnectionTestStage::DeleteObject.as_str(),
code: None,
message: "连接测试成功".to_owned(),
})
}
async fn upload_file(
&self,
config: &BackupStorageConfig,
request: UploadObjectRequest,
) -> Result<(), BackupError> {
let client = self.client(config);
let file_size = tokio::fs::metadata(&request.source)
.await
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?
.len();
if file_size == 0 {
return Err(BackupError::new(
code::S3_UPLOAD_FAILED,
"暂存归档为空,拒绝上传".to_owned(),
));
}
let created_at = request.metadata.created_at.to_rfc3339();
let create = client
.create_multipart_upload()
.bucket(&config.bucket)
.key(&request.object_key)
.metadata("backup-id", &request.metadata.backup_id)
.metadata("sha256", &request.metadata.sha256)
.metadata("created-at", &created_at)
.send()
.await
.map_err(map_s3_error)?;
let upload_id = create
.upload_id()
.ok_or_else(|| {
BackupError::new(
code::S3_UPLOAD_FAILED,
"S3 未返回 multipart upload id".to_owned(),
)
})?
.to_owned();
let upload = upload_parts(&client, config, &request, upload_id.clone(), file_size);
let outcome = tokio::select! {
result = upload => result,
_ = request.cancellation.cancelled() => {
Err(BackupError::new(code::CANCELLED, "上传已取消".to_owned()))
}
};
if let Err(error) = &outcome {
let _ = client
.abort_multipart_upload()
.bucket(&config.bucket)
.key(&request.object_key)
.upload_id(&upload_id)
.send()
.await;
if error.code() != code::CANCELLED {
let _ = client
.delete_object()
.bucket(&config.bucket)
.key(&request.object_key)
.send()
.await;
}
}
outcome
}
async fn head_object(
&self,
config: &BackupStorageConfig,
object_key: &str,
) -> Result<Option<BackupObjectMetadata>, BackupError> {
let client = self.client(config);
match client
.head_object()
.bucket(&config.bucket)
.key(object_key)
.send()
.await
{
Ok(head) => {
let size_bytes = head.content_length().filter(|&value| value >= 0);
let metadata = head.metadata().cloned().unwrap_or_default();
let sha256 = metadata.get("sha256").cloned().ok_or_else(|| {
BackupError::new(
code::REMOTE_VERIFICATION_FAILED,
"远端对象缺少 sha256 metadata".to_owned(),
)
})?;
let backup_id = metadata.get("backup-id").cloned().unwrap_or_default();
let created_at = metadata
.get("created-at")
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
.map(|value| value.with_timezone(&Utc))
.unwrap_or_else(Utc::now);
let Some(size_bytes) = size_bytes else {
return Err(BackupError::new(
code::REMOTE_VERIFICATION_FAILED,
"远端对象缺少大小".to_owned(),
));
};
Ok(Some(BackupObjectMetadata::new(
backup_id,
sha256,
created_at,
size_bytes as u64,
)))
}
Err(error) if is_not_found(&error) => Ok(None),
Err(error) => Err(map_s3_error(error)),
}
}
async fn delete_object(
&self,
config: &BackupStorageConfig,
object_key: &str,
) -> Result<(), BackupError> {
let client = self.client(config);
client
.delete_object()
.bucket(&config.bucket)
.key(object_key)
.send()
.await
.map(|_| ())
.map_err(map_s3_error)
}
async fn presigned_download(
&self,
config: &BackupStorageConfig,
object_key: &str,
file_name: &str,
ttl: Duration,
) -> Result<String, BackupError> {
let client = self.client(config);
let presigning = PresigningConfig::expires_in(ttl)
.map_err(|_| BackupError::new(code::INVALID_CONFIG, "下载地址有效期非法".to_owned()))?;
let request = client
.get_object()
.bucket(&config.bucket)
.key(object_key)
.response_content_disposition(format!("attachment; filename=\"{file_name}\""))
.presigned(presigning)
.await
.map_err(map_s3_error)?;
Ok(request.uri().to_owned())
}
}
/// 并发分片上传 + CompleteMultipartUpload。
async fn upload_parts(
client: &S3Client,
config: &BackupStorageConfig,
request: &UploadObjectRequest,
upload_id: String,
file_size: u64,
) -> Result<(), BackupError> {
let total_parts = file_size.div_ceil(u64::try_from(PART_SIZE).unwrap_or(u64::MAX));
let next_part = Arc::new(AtomicU64::new(1));
let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
let (sender, mut receiver) = tokio::sync::mpsc::channel::<(i32, String)>(CONCURRENCY * 2);
let mut workers = Vec::with_capacity(CONCURRENCY);
for _ in 0..CONCURRENCY {
let client = client.clone();
let semaphore = Arc::clone(&semaphore);
let next_part = Arc::clone(&next_part);
let sender = sender.clone();
let source = request.source.clone();
let bucket = config.bucket.clone();
let key = request.object_key.clone();
let upload_id = upload_id.clone();
workers.push(tokio::spawn(async move {
loop {
let part = next_part.fetch_add(1, Ordering::Relaxed);
if part > total_parts {
break;
}
let _permit = semaphore.acquire().await.map_err(|_| {
BackupError::new(code::S3_UPLOAD_FAILED, "上传并发闸门关闭".to_owned())
})?;
let offset = (part - 1) * (PART_SIZE as u64);
let bytes = read_part(&source, offset, PART_SIZE).await?;
let part_number = i32::try_from(part).map_err(|_| {
BackupError::new(code::S3_UPLOAD_FAILED, "分片号溢出".to_owned())
})?;
let response = client
.upload_part()
.bucket(&bucket)
.key(&key)
.upload_id(&upload_id)
.part_number(part_number)
.body(ByteStream::from(bytes))
.send()
.await
.map_err(map_s3_error)?;
let etag = response
.e_tag()
.ok_or_else(|| {
BackupError::new(code::S3_UPLOAD_FAILED, "S3 未返回分片 ETag".to_owned())
})?
.to_owned();
if sender.send((part_number, etag)).await.is_err() {
break;
}
}
Ok::<(), BackupError>(())
}));
}
drop(sender);
let mut parts = Vec::new();
while let Some((part, etag)) = receiver.recv().await {
parts.push((part, etag));
}
for worker in workers {
worker.await.map_err(|_| {
BackupError::new(code::S3_UPLOAD_FAILED, "上传分片任务崩溃".to_owned())
})??;
}
parts.sort_by_key(|(part, _)| *part);
let completed = CompletedMultipartUpload::builder()
.set_parts(Some(
parts
.into_iter()
.map(|(part, etag)| {
CompletedPart::builder()
.part_number(part)
.e_tag(etag)
.build()
})
.collect(),
))
.build();
client
.complete_multipart_upload()
.bucket(&config.bucket)
.key(&request.object_key)
.upload_id(&upload_id)
.multipart_upload(completed)
.send()
.await
.map(|_| ())
.map_err(map_s3_error)
}
/// 从文件中读取一个分片(最多 `PART_SIZE` 字节)。
async fn read_part(source: &Path, offset: u64, max_len: usize) -> Result<Vec<u8>, BackupError> {
let mut file = tokio::fs::File::open(source)
.await
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "打开暂存归档失败".to_owned()))?;
file.seek(SeekFrom::Start(offset))
.await
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "定位暂存归档失败".to_owned()))?;
let mut buffer = vec![0_u8; max_len];
let n = file
.read(&mut buffer)
.await
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?;
buffer.truncate(n);
Ok(buffer)
}
async fn best_effort_delete(
client: &S3Client,
config: &BackupStorageConfig,
object_key: &str,
) -> Result<(), BackupError> {
client
.delete_object()
.bucket(&config.bucket)
.key(object_key)
.send()
.await
.map(|_| ())
.map_err(map_s3_error)
}
/// 探测失败结果。
fn failed(
stage: ConnectionTestStage,
error: &SdkError<impl std::fmt::Debug + ProvideErrorMetadata, HttpResponse>,
) -> ConnectionTestResult {
let result = ConnectionTestResult {
ok: false,
stage: stage.as_str(),
code: Some(s3_error_code(error)),
message: s3_error_message(error),
};
tracing::warn!(
stage = result.stage,
code = result.code.unwrap_or_default(),
message = %result.message,
"S3 连接探针失败"
);
result
}
/// 稳定错误码。
fn s3_error_code<E: ProvideErrorMetadata>(error: &SdkError<E, HttpResponse>) -> &'static str {
match status_of(error) {
Some(401) => code::S3_AUTH_FAILED,
Some(403) => code::S3_PERMISSION_DENIED,
Some(404) => code::UPSTREAM_FAILURE,
Some(500..=599) => code::UPSTREAM_FAILURE,
_ => code::UPSTREAM_FAILURE,
}
}
fn s3_error_message<E: ProvideErrorMetadata>(error: &SdkError<E, HttpResponse>) -> String {
// 只取稳定的 S3 错误码(如 AccessDenied / NoSuchBucket / SignatureDoesNotMatch),
// 不拼入原始 message,避免泄露请求细节。
let detail = error.code().unwrap_or("unknown");
match status_of(error) {
Some(401) => format!("当前凭据认证失败({detail})"),
Some(403) => format!("当前凭据没有目标前缀的读写权限({detail})"),
Some(404) => format!("目标存储或对象不存在({detail})"),
Some(500..=599) => format!("上游对象存储服务返回失败响应({detail})"),
_ => format!("对象存储请求失败({detail})"),
}
}
/// 提取 HTTP 状态码;非 ServiceError 返回 `None`。
fn status_of<E>(error: &SdkError<E, HttpResponse>) -> Option<u16> {
match error {
SdkError::ServiceError(service) => Some(service.raw().status().as_u16()),
_ => None,
}
}
/// 对象不存在的判定(HeadObject 的 404)。
fn is_not_found<E>(error: &SdkError<E, HttpResponse>) -> bool {
status_of(error) == Some(404)
}
fn map_s3_error<E: ProvideErrorMetadata>(error: SdkError<E, HttpResponse>) -> BackupError {
BackupError::new(s3_error_code(&error), s3_error_message(&error))
}
@@ -0,0 +1,104 @@
//! 受控暂存目录:权限校验、磁盘空间检查与原子改名。
use std::path::{Path, PathBuf};
use crate::{StoreError, StoreResult};
/// 单任务暂存归档上限(默认 64 GiB)。
pub const DEFAULT_MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024 * 1024;
/// 暂存目录所需的最小剩余空间;与单归档上限解耦,避免小磁盘环境被 64 GiB 门槛误拒。
const MIN_STAGING_FREE_BYTES: u64 = 1024 * 1024 * 1024;
/// 备份暂存区。只负责文件系统;归档校验与上传由调用方负责。
#[derive(Debug, Clone)]
pub struct StagingArea {
base_dir: PathBuf,
max_archive_bytes: u64,
}
impl StagingArea {
/// 创建暂存区并确保目录存在、权限为仅运行用户可读写(0700)。
///
/// # Errors
///
/// 目录创建或权限设置失败时返回 [`StoreError`]。
pub fn open(base_dir: PathBuf, max_archive_bytes: u64) -> StoreResult<Self> {
std::fs::create_dir_all(&base_dir).map_err(|_| unavailable("create staging directory"))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt as _;
std::fs::set_permissions(&base_dir, std::fs::Permissions::from_mode(0o700))
.map_err(|_| unavailable("set staging directory permissions"))?;
}
let metadata = std::fs::metadata(&base_dir)
.map_err(|_| unavailable("read staging directory metadata"))?;
if !metadata.is_dir() {
return Err(invalid("staging path is not a directory"));
}
Ok(Self {
base_dir,
max_archive_bytes,
})
}
/// 未完成归档的暂存路径。
#[must_use]
pub fn partial_path(&self, backup_id: &str) -> PathBuf {
self.base_dir.join(format!("{backup_id}.partial"))
}
/// 完成归档的暂存路径。
#[must_use]
pub fn final_path(&self, backup_id: &str) -> PathBuf {
self.base_dir.join(format!("{backup_id}.dump"))
}
/// 单任务暂存归档上限。
#[must_use]
pub const fn max_archive_bytes(&self) -> u64 {
self.max_archive_bytes
}
/// 检查剩余磁盘空间是否足够暂存一个归档。
///
/// 只要求保留一个基本工作余量;单个归档的硬上限由
/// [`Self::max_archive_bytes`] 在写入时兜底。
///
/// # Errors
///
/// 剩余空间不足或读取失败时返回 [`crate::StoreError`]。
pub fn ensure_capacity(&self) -> StoreResult<()> {
let free = fs2::available_space(&self.base_dir)
.map_err(|_| unavailable("read staging free space"))?;
if free < MIN_STAGING_FREE_BYTES {
return Err(StoreError::InvalidData {
entity: "backup staging",
message: "staging disk space is below 1 GiB".to_owned(),
});
}
Ok(())
}
/// 清理该备份在暂存区的全部文件;不存在视为成功。
pub fn cleanup(&self, backup_id: &str) {
let _ = std::fs::remove_file(self.partial_path(backup_id));
let _ = std::fs::remove_file(self.final_path(backup_id));
}
/// 校验一个已完成归档的暂存路径归属本暂存区。
#[must_use]
pub fn owns(&self, path: &Path) -> bool {
path.parent() == Some(self.base_dir.as_path())
}
}
fn unavailable(operation: &'static str) -> StoreError {
crate::postgres_unavailable(operation)
}
fn invalid(message: &str) -> StoreError {
StoreError::InvalidData {
entity: "backup staging",
message: message.to_owned(),
}
}
+208
View File
@@ -0,0 +1,208 @@
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
use gateway_core::engine::credential::ProviderAccountStore;
use super::*;
/// 已完成连接、迁移与 hydration 的 Store 能力集合。
pub struct StoreBundle {
admin_ports: AdminStorePorts,
core_ports: CoreStorePorts,
provider_ports: ProviderStorePorts,
worker_leader_lease: Arc<dyn WorkerLeaderLeasePort>,
health_probes: Vec<Arc<dyn HealthProbe>>,
worker_contributions: Vec<WorkerContribution>,
}
impl StoreBundle {
#[must_use]
pub fn admin_ports(&self) -> AdminStorePorts {
self.admin_ports.clone()
}
#[must_use]
pub fn core_ports(&self) -> CoreStorePorts {
self.core_ports.clone()
}
#[must_use]
pub fn provider_ports(&self) -> ProviderStorePorts {
self.provider_ports.clone()
}
#[must_use]
pub fn worker_leader_lease(&self) -> Arc<dyn WorkerLeaderLeasePort> {
Arc::clone(&self.worker_leader_lease)
}
#[must_use]
pub fn health_probes(&self) -> Vec<Arc<dyn HealthProbe>> {
self.health_probes.clone()
}
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
std::mem::take(&mut self.worker_contributions)
}
}
/// 在返回 Bundle 前完成全部 Store 启动屏障。
pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
const REDIS_NAMESPACE: &str = "codex-proxy-rs";
config.resolve_and_validate(std::path::Path::new("."))?;
let pool = postgres::connect_and_migrate(&config.database_url()?, config.pool).await?;
let redis_client = ::redis::Client::open(config.redis_url()?)
.map_err(|_| redis_unavailable("create Redis client"))?;
let redis_connection = redis_client
.get_connection_manager()
.await
.map_err(|_| redis_unavailable("connect Redis manager"))?;
let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone()));
let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?);
let account_store: Arc<dyn ProviderAccountStore> = provider_accounts;
let credential_leases =
redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?;
let provider_leases = Arc::new(redis::RedisProviderLeaseCoordinator::new(
credential_leases.clone(),
));
let provider_session_affinity = Arc::new(redis::RedisProviderSessionAffinityRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?);
let credential_state = Arc::new(redis::RedisCredentialStateRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?);
let runtime_policy = Arc::new(postgres::PgRuntimeSettingsRepository::new(pool.clone()));
let oauth_pending = Arc::new(redis::RedisOAuthPendingFlowRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?);
let admin_ports = AdminStorePorts::new(
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
Arc::new(AdminAuthStoreAdapter {
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
state: redis::RedisAdminAuthStateRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?,
}),
Arc::new(postgres::PgAdminClientKeyStore::new(pool.clone())),
Arc::new(postgres::PgAdminObservabilityStore::new(
pool.clone(),
Some(credential_leases.clone()),
)),
Arc::new(AdminSettingsStoreAdapter {
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
}),
backup_ports(pool.clone(), &config)?,
);
let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone()));
let (execution, execution_writer) =
postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository));
let execution = Arc::new(execution);
let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone()));
let admissions: Arc<dyn gateway_core::engine::admission::ClientAdmissionPort> = Arc::new(
redis::RedisClientAdmissionRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?,
);
let circuits: Arc<dyn gateway_core::engine::execution::ProviderCircuitPort> =
Arc::new(redis::RedisProviderCircuitRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
gateway_core::engine::execution::ProviderCircuitPolicy::default(),
)?);
// Continuation affinity 是下一轮请求的路由事实,Core 必须直接等待 Redis 确认。
let continuation: Arc<dyn gateway_core::engine::continuation::NativeContinuationPort> =
Arc::new(redis::RedisNativeContinuationRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?);
let (admissions, admission_release_writer) =
redis::BufferedClientAdmissionPort::new(admissions);
let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits);
let core_ports = CoreStorePorts::new(
execution,
(
Arc::new(admissions),
Arc::new(postgres::PgClientAdmissionRecoveryRepository::new(
pool.clone(),
)),
),
Arc::new(circuits),
continuation,
(
Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())),
Arc::new(redis::RedisRuntimeChangeRepository::new(
redis_client,
REDIS_NAMESPACE,
)?),
),
Arc::new(postgres::PgClientApiKeyUsageSink::new(pool.clone())),
);
let provider_ports = ProviderStorePorts::new(
account_store,
provider_leases,
provider_session_affinity,
Arc::new(redis::RedisProviderSessionExclusionRepository::new(
redis_connection.clone(),
REDIS_NAMESPACE,
)?),
credential_state.clone(),
credential_state,
cooldowns,
runtime_policy,
oauth_pending,
);
let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new(
credential_leases,
));
let health_probes: Vec<Arc<dyn HealthProbe>> = vec![
Arc::new(PostgresHealthProbe { pool: pool.clone() }),
Arc::new(RedisHealthProbe {
connection: redis_connection,
}),
];
let worker_contributions = store_worker_contributions(
execution_repository,
execution_writer,
admission_release_writer,
circuit_feedback_writer,
retention,
)?;
Ok(StoreBundle {
admin_ports,
core_ports,
provider_ports,
worker_leader_lease,
health_probes,
worker_contributions,
})
}
/// 构造备份控制面的仓储、导出器与对象存储适配器。
pub(crate) fn backup_ports(
pool: sqlx::PgPool,
config: &StoreConfig,
) -> StoreResult<BackupStorePorts> {
let staging = Arc::new(backup::staging::StagingArea::open(
std::path::PathBuf::from(".runtime/data/backup-staging"),
backup::staging::DEFAULT_MAX_ARCHIVE_BYTES,
)?);
let repository = Arc::new(postgres::PgBackupRepository::new(pool));
let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new(
staging,
config.database.url.clone(),
config.database.password.clone(),
));
let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new());
Ok(BackupStorePorts::new(repository, dump, object_store))
}
+141
View File
@@ -0,0 +1,141 @@
//! Store 启动配置、环境变量解析与校验。
use super::*;
pub(crate) const DATABASE_URL_ENV: &str = "CPR_DATABASE_URL";
pub(crate) const REDIS_URL_ENV: &str = "CPR_REDIS_URL";
pub(crate) const DATABASE_PASSWORD_ENV: &str = "CPR_DATABASE_PASSWORD";
pub(crate) const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD";
/// Store 自己拥有并校验的启动配置。
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StoreConfig {
pub(crate) database: StoreConnectionConfig,
pub(crate) redis: StoreConnectionConfig,
#[serde(default)]
pub(crate) pool: StorePoolConfig,
}
/// PostgreSQL 连接池预算;acquire 超时决定池耗尽时快速失败而非排队积压。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct StorePoolConfig {
pub max_connections: u32,
pub acquire_timeout_seconds: u64,
}
impl Default for StorePoolConfig {
fn default() -> Self {
Self {
max_connections: 10,
acquire_timeout_seconds: 5,
}
}
}
impl StorePoolConfig {
fn validate(&self) -> StoreResult<()> {
if self.max_connections == 0 || self.acquire_timeout_seconds == 0 {
return Err(StoreError::InvalidData {
entity: "store config",
message: "pool limits must be positive".to_owned(),
});
}
Ok(())
}
}
impl StoreConfig {
pub fn resolve_and_validate(&mut self, _source_dir: &std::path::Path) -> StoreResult<()> {
if let Some(url) = optional_environment_value(DATABASE_URL_ENV)? {
self.database.url = url;
}
if let Some(url) = optional_environment_value(REDIS_URL_ENV)? {
self.redis.url = url;
}
if let Some(password) = optional_environment_value(DATABASE_PASSWORD_ENV)? {
self.database.password = password;
}
if let Some(password) = optional_environment_value(REDIS_PASSWORD_ENV)? {
self.redis.password = password;
}
self.database.validate("database")?;
self.redis.validate("redis")?;
self.pool.validate()?;
Ok(())
}
pub(crate) fn database_url(&self) -> StoreResult<String> {
self.database.connection_url("database")
}
pub(crate) fn redis_url(&self) -> StoreResult<String> {
self.redis.connection_url("redis")
}
}
pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult<Option<String>> {
match std::env::var(name) {
Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData {
entity: "store config",
message: format!("environment variable {name} is empty"),
}),
Ok(value) => Ok(Some(value)),
Err(std::env::VarError::NotPresent) => Ok(None),
Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData {
entity: "store config",
message: format!("environment variable {name} is not Unicode"),
}),
}
}
impl fmt::Debug for StoreConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("StoreConfig")
.field("database", &"[REDACTED]")
.field("redis", &"[REDACTED]")
.finish()
}
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct StoreConnectionConfig {
pub(crate) url: String,
pub(crate) password: String,
}
impl StoreConnectionConfig {
fn validate(&self, field: &'static str) -> StoreResult<()> {
require_nonempty("store config", field, &self.url)?;
if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit())
{
return Err(StoreError::InvalidData {
entity: "store config",
message: format!("{field}.password must be exactly 48 hexadecimal characters"),
});
}
self.connection_url(field).map(|_| ())
}
fn connection_url(&self, field: &'static str) -> StoreResult<String> {
let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData {
entity: "store config",
message: format!("{field}.url is invalid"),
})?;
if url.password().is_some() {
return Err(StoreError::InvalidData {
entity: "store config",
message: format!("{field}.url must not contain a password"),
});
}
url.set_password(Some(&self.password))
.map_err(|()| StoreError::InvalidData {
entity: "store config",
message: format!("{field}.url cannot carry credentials"),
})?;
Ok(url.to_string())
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,870 @@
//! `backup_settings` 与 `backup_records` 的 PostgreSQL owner。
//!
//! 实现 `gateway-admin::ports::backup::BackupRepository`。本模块只执行事务与
//! 条件更新,不决定保留策略或状态机语义。
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use secrecy::{ExposeSecret as _, SecretString};
use sqlx::{PgPool, Postgres, Row as _, Transaction};
use gateway_admin::model::backup::{
BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupRecordSeed, BackupSettings,
BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
};
use gateway_admin::model::{MutationContext, Revision as AdminRevision};
use gateway_admin::ports::backup::{BackupRepository, StatusTransitionUpdate};
use gateway_admin::ports::store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult};
use crate::{
StoreError, StoreResult,
postgres::{
admin_security_audit::AdminAuditEvent,
runtime_settings::bump_config_revision_in_transaction,
},
};
/// `backup_records` 的最大分页大小。
const BACKUP_PAGE_LIMIT: u32 = 200;
/// 备份任务/配置仓储。
#[derive(Clone)]
pub struct PgBackupRepository {
pool: PgPool,
}
impl PgBackupRepository {
/// 包装连接池。
#[must_use]
pub const fn new(pool: PgPool) -> Self {
Self { pool }
}
}
#[async_trait]
impl BackupRepository for PgBackupRepository {
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
load_backup_settings(&self.pool).await
}
async fn update_storage_settings(
&self,
command: UpdateBackupStorageCommand,
context: &MutationContext,
) -> AdminStoreResult<(BackupSettings, AdminRevision)> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| store_unavailable("begin backup storage update"))?;
let result = async {
let current = lock_settings_in_transaction(&mut transaction).await?;
ensure_storage_identity_stable(&current, &command, &mut transaction).await?;
let secret = command
.secret_access_key
.as_ref()
.map(|secret| secret.expose_secret().to_owned());
sqlx::query(
"update backup_settings
set endpoint = $1,
region = $2,
bucket = $3,
access_key_id = $4,
secret_access_key = coalesce($5, secret_access_key),
prefix = $6,
force_path_style = $7,
storage_revision = storage_revision + 1,
last_verified_at = null,
updated_at = now()
where id = 1",
)
.bind(&command.endpoint)
.bind(&command.region)
.bind(&command.bucket)
.bind(&command.access_key_id)
.bind(secret.as_deref())
.bind(&command.prefix)
.bind(command.force_path_style)
.execute(&mut *transaction)
.await
.map_err(|_| store_unavailable("update backup storage"))?;
let store_revision = bump_config_revision_in_transaction(&mut transaction)
.await
.map_err(map_admin_error)?;
let revision = admin_revision(store_revision)?;
let changed = storage_changed_fields(&current, &command);
let audit = crate::mutation_audit(
context,
"backup.s3_config_updated",
"backup_settings",
"1",
changed,
);
append_audit_in_transaction(&mut transaction, audit, Some(revision))
.await
.map_err(map_admin_error)?;
let settings = load_settings_in_transaction(&mut transaction).await?;
Ok((settings, revision))
}
.await;
match result {
Ok(result) => {
transaction
.commit()
.await
.map_err(|_| store_unavailable("commit backup storage update"))?;
Ok(result)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| store_unavailable("rollback backup storage update"))?;
Err(error)
}
}
}
async fn update_schedule_settings(
&self,
command: UpdateBackupScheduleCommand,
next_run_at: Option<DateTime<Utc>>,
context: &MutationContext,
) -> AdminStoreResult<BackupSettings> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| store_unavailable("begin backup schedule update"))?;
let result = async {
sqlx::query(
"update backup_settings
set schedule_enabled = $1,
cron_expression = $2,
schedule_timezone = $3,
retention_days = $4,
retention_count = $5,
next_run_at = $6,
updated_at = now()
where id = 1",
)
.bind(command.schedule_enabled)
.bind(&command.cron_expression)
.bind(&command.schedule_timezone)
.bind(i64::from(command.retention_days))
.bind(i64::from(command.retention_count))
.bind(next_run_at)
.execute(&mut *transaction)
.await
.map_err(|_| store_unavailable("update backup schedule"))?;
let audit = crate::mutation_audit(
context,
"backup.schedule_updated",
"backup_settings",
"1",
vec![
"schedule_enabled".to_owned(),
"cron_expression".to_owned(),
"schedule_timezone".to_owned(),
"retention_days".to_owned(),
"retention_count".to_owned(),
],
);
append_audit_in_transaction(&mut transaction, audit, None)
.await
.map_err(map_admin_error)?;
load_settings_in_transaction(&mut transaction).await
}
.await;
match result {
Ok(settings) => {
transaction
.commit()
.await
.map_err(|_| store_unavailable("commit backup schedule update"))?;
Ok(settings)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| store_unavailable("rollback backup schedule update"))?;
Err(error)
}
}
}
async fn record_verification(
&self,
storage_revision: u64,
at: DateTime<Utc>,
) -> AdminStoreResult<bool> {
let result = sqlx::query(
"update backup_settings
set last_verified_at = $2, updated_at = now()
where id = 1 and storage_revision = $1",
)
.bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?)
.bind(at)
.execute(&self.pool)
.await
.map_err(|_| store_unavailable("record backup storage verification"))?;
Ok(result.rows_affected() == 1)
}
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| store_unavailable("begin backup record insert"))?;
let result = async {
let _ = lock_settings_in_transaction(&mut transaction).await?;
insert_queued_in_transaction(&mut transaction, &seed).await
}
.await;
match result {
Ok(record) => {
transaction
.commit()
.await
.map_err(|_| store_unavailable("commit backup record insert"))?;
Ok(record)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| store_unavailable("rollback backup record insert"))?;
Err(error)
}
}
}
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| store_unavailable("begin scheduled insert"))?;
match insert_queued_in_transaction(&mut transaction, &seed).await {
Ok(_) => {
transaction
.commit()
.await
.map_err(|_| store_unavailable("commit scheduled insert"))?;
Ok(true)
}
Err(error) if is_active_or_scheduled_conflict(&error) => {
transaction
.rollback()
.await
.map_err(|_| store_unavailable("rollback scheduled insert"))?;
Ok(false)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| store_unavailable("rollback scheduled insert"))?;
Err(error)
}
}
}
async fn list_backup_records(
&self,
query: BackupRecordListQuery,
) -> AdminStoreResult<BackupRecordPage> {
let limit = u32::from(query.page_size.get()).min(BACKUP_PAGE_LIMIT);
let offset = u64::from(query.page.saturating_sub(1)) * u64::from(limit);
let status = query.status.map(BackupStatus::as_str);
let trigger = query.trigger.map(BackupTriggerKind::as_str);
let total = sqlx::query_scalar::<_, i64>(
"select count(*) from backup_records
where ($1::text is null or status = $1)
and ($2::text is null or trigger_kind = $2)",
)
.bind(status)
.bind(trigger)
.fetch_one(&self.pool)
.await
.map_err(|_| store_unavailable("count backup records"))?;
let rows = sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records
where ($1::text is null or status = $1)
and ($2::text is null or trigger_kind = $2)
order by created_at desc, id desc
limit $3 offset $4",
)
.bind(status)
.bind(trigger)
.bind(i64::from(limit))
.bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?)
.fetch_all(&self.pool)
.await
.map_err(|_| store_unavailable("list backup records"))?;
let items = rows
.iter()
.map(backup_record_from_row)
.collect::<Result<Vec<_>, _>>()?;
Ok(BackupRecordPage {
items,
total: u64::try_from(total).map_err(|_| store_invalid("record count"))?,
page: query.page,
page_size: query.page_size,
})
}
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>> {
sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records where id = $1",
)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|_| store_unavailable("load backup record"))?
.map(|row| backup_record_from_row(&row))
.transpose()
}
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
let rows = sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records
where status in ('dumping', 'uploading')
order by created_at, id",
)
.fetch_all(&self.pool)
.await
.map_err(|_| store_unavailable("list intermediate backup records"))?;
rows.iter()
.map(backup_record_from_row)
.collect::<Result<Vec<_>, _>>()
}
async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
let limit = limit.clamp(1, 100);
let rows = sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records
where status = 'deleting'
order by created_at, id
limit $1",
)
.bind(i64::from(limit))
.fetch_all(&self.pool)
.await
.map_err(|_| store_unavailable("list pending deletions"))?;
rows.iter()
.map(backup_record_from_row)
.collect::<Result<Vec<_>, _>>()
}
async fn list_expired_records(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
let limit = limit.clamp(1, 1000);
let now = Utc::now();
let rows = sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records
where expires_at is not null
and expires_at <= $1
and status in ('completed', 'failed')
order by created_at, id
limit $2",
)
.bind(now)
.bind(i64::from(limit))
.fetch_all(&self.pool)
.await
.map_err(|_| store_unavailable("list expired backups"))?;
rows.iter()
.map(backup_record_from_row)
.collect::<Result<Vec<_>, _>>()
}
async fn claim_next_queued(
&self,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
sqlx::query(
"with claimed as (
select backup_records.id from backup_records
where backup_records.status = 'queued'
order by backup_records.created_at, backup_records.id
limit 1
for update skip locked
)
update backup_records br
set status = 'dumping',
started_at = $1,
completed_at = null,
attempt_count = br.attempt_count + 1,
updated_at = $1
from claimed
where br.id = claimed.id
returning br.id, br.trigger_kind, br.status, br.scheduled_at, br.object_key,
br.size_bytes, br.sha256, br.attempt_count, br.error_code,
br.error_message, br.started_at, br.completed_at, br.expires_at,
br.created_at, br.updated_at",
)
.bind(now)
.fetch_optional(&self.pool)
.await
.map_err(|_| store_unavailable("claim queued backup task"))?
.map(|row| backup_record_from_row(&row))
.transpose()
}
async fn transition_status(
&self,
id: &str,
from: BackupStatus,
to: BackupStatus,
update: StatusTransitionUpdate,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
let size_bytes = update
.size_bytes
.map(i64::try_from)
.transpose()
.map_err(|_| store_invalid("size bytes"))?;
sqlx::query(
"update backup_records
set status = $1,
size_bytes = coalesce($2, size_bytes),
sha256 = coalesce($3, sha256),
error_code = coalesce($4, error_code),
error_message = coalesce($5, error_message),
completed_at = coalesce(
$6,
case when $1 in ('completed', 'failed') then $7 else completed_at end
),
updated_at = $7
where id = $8 and status = $9
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at",
)
.bind(to.as_str())
.bind(size_bytes)
.bind(update.sha256.as_deref())
.bind(update.error_code.as_deref())
.bind(update.error_message.as_deref())
.bind(update.completed_at)
.bind(now)
.bind(id)
.bind(from.as_str())
.fetch_optional(&self.pool)
.await
.map_err(|_| store_unavailable("transition backup task"))?
.map(|row| backup_record_from_row(&row))
.transpose()
}
async fn transition_to_deleting(
&self,
id: &str,
now: DateTime<Utc>,
) -> AdminStoreResult<Option<BackupRecord>> {
sqlx::query(
"update backup_records
set status = 'deleting', updated_at = $1
where id = $2 and status in ('completed', 'failed')
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at",
)
.bind(now)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|_| store_unavailable("transition backup to deleting"))?
.map(|row| backup_record_from_row(&row))
.transpose()
}
async fn delete_record(&self, id: &str) -> AdminStoreResult<()> {
sqlx::query("delete from backup_records where id = $1")
.bind(id)
.execute(&self.pool)
.await
.map(|_| ())
.map_err(|_| store_unavailable("delete backup record"))
}
async fn advance_schedule_cursor(
&self,
next_run_at: DateTime<Utc>,
expected_cron: &str,
expected_timezone: &str,
) -> AdminStoreResult<bool> {
let result = sqlx::query(
"update backup_settings
set next_run_at = $1
where id = 1
and schedule_enabled
and cron_expression = $2
and schedule_timezone = $3",
)
.bind(next_run_at)
.bind(expected_cron)
.bind(expected_timezone)
.execute(&self.pool)
.await
.map_err(|_| store_unavailable("advance backup schedule cursor"))?;
Ok(result.rows_affected() == 1)
}
async fn list_scheduled_completed_desc(
&self,
limit: u32,
) -> AdminStoreResult<Vec<BackupRecord>> {
let limit = limit.clamp(1, 10000);
let rows = sqlx::query(
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at
from backup_records
where trigger_kind = 'scheduled' and status = 'completed'
order by completed_at desc, id desc
limit $1",
)
.bind(i64::from(limit))
.fetch_all(&self.pool)
.await
.map_err(|_| store_unavailable("list scheduled completed backups"))?;
rows.iter()
.map(backup_record_from_row)
.collect::<Result<Vec<_>, _>>()
}
}
/// Store 内部 Revision → Admin Revision 转换。
fn admin_revision(revision: crate::Revision) -> AdminStoreResult<AdminRevision> {
AdminRevision::new(revision.get()).map_err(|_| {
AdminStoreError::new(AdminStoreErrorKind::Invalid, "revision", "zero revision")
})
}
/// 锁定并读取单例配置行。
async fn lock_settings_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
) -> AdminStoreResult<BackupSettings> {
sqlx::query(
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
retention_days, retention_count, next_run_at, last_verified_at, updated_at
from backup_settings where id = 1 for update",
)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| store_unavailable("lock backup settings"))?
.map(|row| backup_settings_from_row(&row))
.transpose()?
.ok_or_else(|| store_not_found("backup settings"))
}
async fn load_settings_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
) -> AdminStoreResult<BackupSettings> {
sqlx::query(
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
retention_days, retention_count, next_run_at, last_verified_at, updated_at
from backup_settings where id = 1",
)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| store_unavailable("load backup settings in transaction"))?
.map(|row| backup_settings_from_row(&row))
.transpose()?
.ok_or_else(|| store_not_found("backup settings"))
}
async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult<BackupSettings> {
sqlx::query(
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
retention_days, retention_count, next_run_at, last_verified_at, updated_at
from backup_settings where id = 1",
)
.fetch_optional(pool)
.await
.map_err(|_| store_unavailable("load backup settings"))?
.map(|row| backup_settings_from_row(&row))
.transpose()?
.ok_or_else(|| store_not_found("backup settings"))
}
/// 插入 queued 记录;返回记录或唯一约束冲突。
async fn insert_queued_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
seed: &BackupRecordSeed,
) -> AdminStoreResult<BackupRecord> {
let row = sqlx::query(
"insert into backup_records (
id, trigger_kind, status, scheduled_at, object_key, expires_at, created_at, updated_at
) values ($1, $2, 'queued', $3, $4, $5, now(), now())
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
created_at, updated_at",
)
.bind(&seed.id)
.bind(seed.trigger_kind.as_str())
.bind(seed.scheduled_at)
.bind(&seed.object_key)
.bind(seed.expires_at)
.fetch_one(&mut **transaction)
.await
.map_err(map_insert_error)?;
backup_record_from_row(&row)
}
/// 把唯一约束冲突映射为活跃任务/计划冲突。
fn map_insert_error(error: sqlx::Error) -> AdminStoreError {
match &error {
sqlx::Error::Database(database_error)
if database_error.constraint() == Some("backup_records_active_uq") =>
{
AdminStoreError::new(
AdminStoreErrorKind::Conflict,
"backup record",
"an active backup task already exists",
)
}
sqlx::Error::Database(database_error)
if database_error.constraint() == Some("backup_records_scheduled_uq") =>
{
AdminStoreError::new(
AdminStoreErrorKind::Conflict,
"backup record",
"a scheduled backup already exists at this time",
)
}
_ => AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
"backup record",
"backup record insert failed",
),
}
}
fn is_active_or_scheduled_conflict(error: &AdminStoreError) -> bool {
error.kind() == AdminStoreErrorKind::Conflict
}
/// 已有记录时锁定存储身份:endpoint/region/bucket/path-style 不允许变化。
async fn ensure_storage_identity_stable(
current: &BackupSettings,
command: &UpdateBackupStorageCommand,
transaction: &mut Transaction<'_, Postgres>,
) -> AdminStoreResult<()> {
let identity_changed = current.endpoint.as_deref() != Some(command.endpoint.as_str())
|| current.region.as_deref() != Some(command.region.as_str())
|| current.bucket.as_deref() != Some(command.bucket.as_str())
|| current.force_path_style != command.force_path_style;
if !identity_changed {
return Ok(());
}
let records_exist =
sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)")
.fetch_one(&mut **transaction)
.await
.map_err(|_| store_unavailable("check backup records existence"))?;
if records_exist {
return Err(AdminStoreError::new(
AdminStoreErrorKind::Conflict,
"backup storage",
"storage identity cannot change while backup records exist",
));
}
Ok(())
}
fn storage_changed_fields(
current: &BackupSettings,
command: &UpdateBackupStorageCommand,
) -> Vec<String> {
let mut fields = Vec::new();
if current.endpoint.as_deref() != Some(command.endpoint.as_str()) {
fields.push("endpoint".to_owned());
}
if current.region.as_deref() != Some(command.region.as_str()) {
fields.push("region".to_owned());
}
if current.bucket.as_deref() != Some(command.bucket.as_str()) {
fields.push("bucket".to_owned());
}
if current.access_key_id.as_deref() != Some(command.access_key_id.as_str()) {
fields.push("access_key_id".to_owned());
}
if command.secret_access_key.is_some() {
fields.push("secret_access_key".to_owned());
}
if current.prefix.as_deref() != Some(command.prefix.as_str()) {
fields.push("prefix".to_owned());
}
if current.force_path_style != command.force_path_style {
fields.push("force_path_style".to_owned());
}
fields
}
/// 在同事务追加审计事件;`revision` 为空时不写 config_revision。
async fn append_audit_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
mut event: AdminAuditEvent,
revision: Option<AdminRevision>,
) -> StoreResult<()> {
event.config_revision = revision
.map(|revision| {
i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData {
entity: "backup",
message: "revision overflow".to_owned(),
})
})
.transpose()?;
event.validate()?;
sqlx::query(
"insert into admin_audit_events (
id, actor_kind, actor_admin_user_id, actor_ref, admin_request_id,
action, entity_kind, entity_ref, config_revision, changed_fields, created_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)",
)
.bind(event.id)
.bind(event.actor_kind.as_str())
.bind(event.actor_admin_user_id)
.bind(event.actor_ref)
.bind(event.admin_request_id)
.bind(event.action)
.bind(event.entity_kind)
.bind(event.entity_ref)
.bind(event.config_revision)
.bind(event.changed_fields)
.bind(event.created_at)
.execute(&mut **transaction)
.await
.map_err(|_| crate::postgres_unavailable("append backup audit event"))?;
Ok(())
}
fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<BackupRecord> {
let size_bytes = decode(row.try_get::<Option<i64>, _>("size_bytes"))?
.map(u64::try_from)
.transpose()
.map_err(|_| store_invalid("size_bytes"))?;
Ok(BackupRecord {
id: decode(row.try_get("id"))?,
trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?)
.ok_or_else(|| store_invalid("trigger_kind"))?,
status: BackupStatus::parse(decode(row.try_get::<&str, _>("status"))?)
.ok_or_else(|| store_invalid("status"))?,
scheduled_at: decode(row.try_get("scheduled_at"))?,
object_key: decode(row.try_get("object_key"))?,
size_bytes,
sha256: decode(row.try_get("sha256"))?,
attempt_count: u32::try_from(decode(row.try_get::<i32, _>("attempt_count"))?)
.map_err(|_| store_invalid("attempt_count"))?,
error_code: decode(row.try_get("error_code"))?,
error_message: decode(row.try_get("error_message"))?,
started_at: decode(row.try_get("started_at"))?,
completed_at: decode(row.try_get("completed_at"))?,
expires_at: decode(row.try_get("expires_at"))?,
created_at: decode(row.try_get("created_at"))?,
updated_at: decode(row.try_get("updated_at"))?,
})
}
fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<BackupSettings> {
let storage_revision = u64::try_from(decode(row.try_get::<i64, _>("storage_revision"))?)
.map_err(|_| store_invalid("storage_revision"))?;
let secret = decode(row.try_get::<Option<String>, _>("secret_access_key"))?;
Ok(BackupSettings {
storage_revision,
endpoint: decode(row.try_get("endpoint"))?,
region: decode(row.try_get("region"))?,
bucket: decode(row.try_get("bucket"))?,
access_key_id: decode(row.try_get("access_key_id"))?,
secret_access_key: secret.map(SecretString::from),
prefix: decode(row.try_get("prefix"))?,
force_path_style: decode(row.try_get("force_path_style"))?,
schedule_enabled: decode(row.try_get("schedule_enabled"))?,
cron_expression: decode(row.try_get("cron_expression"))?,
schedule_timezone: decode(row.try_get("schedule_timezone"))?,
retention_days: u32::try_from(decode(row.try_get::<i64, _>("retention_days"))?)
.map_err(|_| store_invalid("retention_days"))?,
retention_count: u32::try_from(decode(row.try_get::<i64, _>("retention_count"))?)
.map_err(|_| store_invalid("retention_count"))?,
next_run_at: decode(row.try_get("next_run_at"))?,
last_verified_at: decode(row.try_get("last_verified_at"))?,
updated_at: decode(row.try_get("updated_at"))?,
})
}
/// sqlx 解码错误统一映射为备份 InvalidData。
fn decode<T>(result: Result<T, sqlx::Error>) -> AdminStoreResult<T> {
result.map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
"backup",
"backup record decode failed",
)
})
}
/// StoreError → AdminStoreError 的显式映射。
fn map_admin_error(error: StoreError) -> AdminStoreError {
match error {
StoreError::Unavailable { .. } => AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
"backup",
"backup store unavailable",
),
StoreError::NotFound { entity, .. } => {
AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found")
}
StoreError::Conflict { entity, .. } => {
AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict")
}
StoreError::InvalidData { entity, message } => {
AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message)
}
}
}
fn store_unavailable(operation: &'static str) -> AdminStoreError {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
"backup",
format!("{operation}: store unavailable"),
)
}
fn store_not_found(entity: &'static str) -> AdminStoreError {
AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found")
}
fn store_invalid(message: &'static str) -> AdminStoreError {
AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", message)
}
@@ -414,12 +414,18 @@ impl ModelRequestRepository for PgExecutionStore {
reasoning_preset, request_kind, subagent_kind, compact,
image_generation_requested, started_at, deadline_at,
provider_kind, provider_account_id, provider_account_ref,
provider_account_name_snapshot, provider_account_email_snapshot,
provider_account_authentication_kind_snapshot,
upstream_model_id, upstream_transport, http_version,
attempt_count, upstream_send_state
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
$13, $14, $15, $16, $17, $18, $19,
$20, $21, $22, $23, $24, $25, 1, 'not_sent'
$20, $21, $22,
(select name from provider_accounts where id = $21),
(select email from provider_accounts where id = $21),
(select authentication_kind from provider_accounts where id = $21),
$23, $24, $25, 1, 'not_sent'
)",
)
.bind(request.id)
@@ -466,6 +472,15 @@ impl ModelRequestRepository for PgExecutionStore {
set provider_kind = $2,
provider_account_id = $3,
provider_account_ref = $4,
provider_account_name_snapshot = (
select name from provider_accounts where id = $3
),
provider_account_email_snapshot = (
select email from provider_accounts where id = $3
),
provider_account_authentication_kind_snapshot = (
select authentication_kind from provider_accounts where id = $3
),
upstream_model_id = $5,
upstream_transport = $6,
http_version = $7,
@@ -1,4 +1,4 @@
//! 八张终态业务表的 PostgreSQL adapters。
//! 九张终态业务表的 PostgreSQL adapters。
use async_trait::async_trait;
use sqlx::{PgPool, postgres::PgPoolOptions};
@@ -9,6 +9,7 @@ use crate::{
mod admin_security_audit;
mod admission_recovery;
mod backup;
mod client_keys;
mod execution;
mod execution_buffer;
@@ -21,6 +22,7 @@ mod snapshot;
pub use admin_security_audit::*;
pub use admission_recovery::*;
pub use backup::*;
pub use client_keys::*;
pub use execution::*;
pub use execution_buffer::*;
@@ -79,6 +81,13 @@ pub trait ControlPlaneRepository: Send + Sync {
replacement: ControlPlaneReplacement,
) -> StoreResult<ControlPlaneSnapshot>;
/// 更新 admin_api_key 字段并推进 config revision。
async fn replace_admin_api_key(
&self,
admin_api_key: Option<String>,
audit: AdminAuditEvent,
) -> StoreResult<Revision>;
async fn create_client_api_key(
&self,
key: NewClientApiKey,
@@ -174,6 +183,15 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
}
}
async fn replace_admin_api_key(
&self,
admin_api_key: Option<String>,
audit: AdminAuditEvent,
) -> StoreResult<Revision> {
self.apply_targeted_mutation(ControlPlaneMutation::SetAdminApiKey(admin_api_key), audit)
.await
}
async fn create_client_api_key(
&self,
key: NewClientApiKey,
@@ -226,6 +244,7 @@ enum ControlPlaneMutation {
UpdateClientApiKey(UpdateClientApiKeyDetails),
SetClientApiKeyEnabled { id: String, enabled: bool },
DeleteClientApiKey(String),
SetAdminApiKey(Option<String>),
}
impl PgControlPlaneRepository {
@@ -255,6 +274,9 @@ impl PgControlPlaneRepository {
ControlPlaneMutation::DeleteClientApiKey(id) => {
delete_client_api_key_in_transaction(&mut transaction, &id).await?;
}
ControlPlaneMutation::SetAdminApiKey(key) => {
update_admin_api_key_in_transaction(&mut transaction, key).await?;
}
}
append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?;
Ok(revision)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,322 @@
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
use super::*;
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
#[derive(Clone)]
pub struct PgObservabilityRepository {
pool: PgPool,
}
impl PgObservabilityRepository {
#[must_use]
pub const fn new(pool: PgPool) -> Self {
Self { pool }
}
}
/// `gateway-admin` 观测端口的 PostgreSQL adapter。
///
/// SQL 查询与内部投影继续由 [`PgObservabilityRepository`] 唯一拥有;本类型只负责
/// Admin UTC 领域模型与持久化投影之间的无格式化转换。
#[derive(Clone)]
pub struct PgAdminObservabilityStore {
repository: PgObservabilityRepository,
runtime_signals: Option<RedisCredentialLeaseRepository>,
}
impl PgAdminObservabilityStore {
#[must_use]
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
Self {
repository: PgObservabilityRepository::new(pool),
runtime_signals,
}
}
}
#[async_trait]
impl ObservabilityRepository for PgObservabilityRepository {
async fn dashboard_summary(
&self,
range: ObservabilityRange,
) -> StoreResult<DashboardObservation> {
let filter = UsageRecordFilter::default();
let account_usage_range = ObservabilityRange::new(
range.end - TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS),
range.end,
)?;
let account_usage_query =
ProviderAccountUsageQuery::recent(account_usage_range, DASHBOARD_ACCOUNT_LIMIT)?
.with_hourly_request_buckets()?;
let recent_query = UsageRecordQuery {
range,
filter: UsageRecordFilter {
outcome: Some("succeeded".to_owned()),
..UsageRecordFilter::default()
},
cursor: None,
page: ObservabilityPageNumber::new(1)?,
page_size: ObservabilityPageSize::new(10)?,
};
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
let (requests, attempts, provider_accounts) = futures::try_join!(
request_metrics(&self.pool, range, &filter),
attempt_metrics(&self.pool, range, &filter),
provider_account_metrics(&self.pool, range.end),
)?;
let (trend, account_usage, recent_requests) = futures::try_join!(
request_metric_series(&self.pool, range, &filter),
provider_account_usage(&self.pool, account_usage_query),
async { Ok(list_usage_records(&self.pool, recent_query).await?.items) },
)?;
Ok(DashboardObservation {
range,
requests,
attempts,
provider_accounts,
trend,
account_usage,
recent_requests,
})
}
async fn dashboard_trend(
&self,
range: ObservabilityRange,
) -> StoreResult<Vec<RequestMetricPoint>> {
request_metric_series(&self.pool, range, &UsageRecordFilter::default()).await
}
async fn usage_trend(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<Vec<RequestMetricPoint>> {
request_metric_series(&self.pool, range, &filter).await
}
async fn usage_calculated_billing_facts(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<Vec<CalculatedUsageBillingFact>> {
calculated_usage_billing_facts(&self.pool, range, &filter).await
}
async fn provider_account_usage(
&self,
query: ProviderAccountUsageQuery,
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
provider_account_usage(&self.pool, query).await
}
async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult<UsageRecordPage> {
list_usage_records(&self.pool, query).await
}
async fn usage_record_detail(&self, request_id: &str) -> StoreResult<UsageRecordDetail> {
usage_record_detail(&self.pool, request_id).await
}
async fn usage_summary(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<UsageOverview> {
filter.validate()?;
let requests = request_metrics(&self.pool, range, &filter).await?;
let attempts = attempt_metrics(&self.pool, range, &filter).await?;
let providers = provider_observations(&self.pool, range, &filter).await?;
Ok(UsageOverview {
range,
requests,
attempts,
providers,
})
}
async fn usage_diagnostics(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
dimension: DiagnosticDimension,
) -> StoreResult<Vec<DiagnosticObservation>> {
usage_diagnostics(&self.pool, range, &filter, dimension).await
}
async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult<OpsErrorPage> {
list_ops_errors(&self.pool, query).await
}
}
#[async_trait]
impl AdminObservabilityStore for PgAdminObservabilityStore {
async fn dashboard_summary(
&self,
range: admin_observability::TimeRange,
) -> AdminStoreResult<admin_observability::DashboardObservation> {
let observation = self
.repository
.dashboard_summary(store_range(range)?)
.await
.map_err(observability_error)?;
admin_dashboard_observation(observation)
}
async fn dashboard_runtime_slots(
&self,
observed_at: DateTime<Utc>,
) -> AdminStoreResult<Option<admin_observability::DashboardRuntimeSlots>> {
let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at)
.await
.map_err(observability_error)?;
let active_accounts = u64::try_from(active_account_ids.len())
.map_err(|_| observability_error(invalid("active account count overflows u64")))?;
if active_account_ids.is_empty() {
return Ok(Some(admin_observability::DashboardRuntimeSlots {
active_accounts,
used_slots: Some(0),
}));
}
let Some(runtime_signals) = &self.runtime_signals else {
return Ok(None);
};
let signals = match runtime_signals
.credential_runtime_signals(&active_account_ids)
.await
{
Ok(signals) => signals,
Err(_) => {
return Ok(Some(admin_observability::DashboardRuntimeSlots {
active_accounts,
used_slots: None,
}));
}
};
let used_slots = signals.into_iter().fold(0_u64, |total, signal| {
total.saturating_add(u64::from(signal.in_flight))
});
Ok(Some(admin_observability::DashboardRuntimeSlots {
active_accounts,
used_slots: Some(used_slots),
}))
}
async fn dashboard_trend(
&self,
range: admin_observability::TimeRange,
) -> AdminStoreResult<Vec<admin_observability::RequestMetricPoint>> {
self.repository
.dashboard_trend(store_range(range)?)
.await
.map_err(observability_error)?
.into_iter()
.map(admin_request_metric_point)
.collect()
}
async fn usage_trend(
&self,
range: admin_observability::TimeRange,
filter: admin_observability::UsageFilter,
) -> AdminStoreResult<Vec<admin_observability::RequestMetricPoint>> {
self.repository
.usage_trend(store_range(range)?, store_usage_filter(filter))
.await
.map_err(observability_error)?
.into_iter()
.map(admin_request_metric_point)
.collect()
}
async fn usage_calculated_billing_facts(
&self,
range: admin_observability::TimeRange,
filter: admin_observability::UsageFilter,
) -> AdminStoreResult<Vec<admin_observability::UsageCalculatedBillingFact>> {
self.repository
.usage_calculated_billing_facts(store_range(range)?, store_usage_filter(filter))
.await
.map_err(observability_error)?
.into_iter()
.map(admin_calculated_usage_billing_fact)
.collect()
}
async fn list_usage_records(
&self,
query: admin_observability::UsageQuery,
) -> AdminStoreResult<admin_observability::UsagePage> {
let page = self
.repository
.list_usage_records(store_usage_query(query)?)
.await
.map_err(observability_error)?;
admin_usage_page(page)
}
async fn usage_record_detail(
&self,
request_id: &str,
) -> AdminStoreResult<admin_observability::UsageDetail> {
let detail = self
.repository
.usage_record_detail(request_id)
.await
.map_err(observability_error)?;
admin_usage_detail(detail)
}
async fn usage_summary(
&self,
range: admin_observability::TimeRange,
filter: admin_observability::UsageFilter,
) -> AdminStoreResult<admin_observability::UsageOverview> {
let overview = self
.repository
.usage_summary(store_range(range)?, store_usage_filter(filter))
.await
.map_err(observability_error)?;
admin_usage_overview(overview)
}
async fn usage_diagnostics(
&self,
range: admin_observability::TimeRange,
filter: admin_observability::UsageFilter,
dimension: admin_observability::DiagnosticDimension,
) -> AdminStoreResult<Vec<admin_observability::DiagnosticObservation>> {
self.repository
.usage_diagnostics(
store_range(range)?,
store_usage_filter(filter),
store_diagnostic_dimension(dimension),
)
.await
.map_err(observability_error)?
.into_iter()
.map(admin_diagnostic_observation)
.collect()
}
async fn list_ops_errors(
&self,
query: admin_observability::OpsErrorQuery,
) -> AdminStoreResult<admin_observability::OpsErrorPage> {
let page = self
.repository
.list_ops_errors(store_ops_error_query(query)?)
.await
.map_err(observability_error)?;
admin_ops_error_page(page)
}
}
pub(crate) fn observability_error(
error: StoreError,
) -> gateway_admin::ports::store::AdminStoreError {
admin_store_error("observability", error)
}
@@ -0,0 +1,864 @@
//! Store 与 Admin 领域之间的无格式化转换及 row 解析。
use super::*;
pub(crate) fn store_range(
range: admin_observability::TimeRange,
) -> AdminStoreResult<ObservabilityRange> {
ObservabilityRange::new(range.start, range.end).map_err(observability_error)
}
pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter {
UsageRecordFilter {
client_api_key_ref: filter.client_api_key_ref,
request_id: filter.request_id,
provider_account_ref: filter.provider_account_ref,
operation: filter.operation,
provider_kind: filter.provider_kind,
model: filter.model,
outcome: filter.outcome.map(store_request_outcome),
status_code: filter.status_code,
transport: filter.transport,
attempt_index: filter.attempt_index,
response_id: filter.response_id,
upstream_request_id: filter.upstream_request_id,
search: filter.search,
}
}
pub(crate) fn store_request_outcome(outcome: admin_observability::RequestOutcome) -> String {
outcome.as_str().to_owned()
}
pub(crate) fn store_cursor(
cursor: admin_observability::ObservabilityCursor,
) -> AdminStoreResult<ObservabilityCursor> {
ObservabilityCursor::new(cursor.observed_at, cursor.stable_id).map_err(observability_error)
}
pub(crate) fn store_usage_query(
query: admin_observability::UsageQuery,
) -> AdminStoreResult<UsageRecordQuery> {
Ok(UsageRecordQuery {
range: store_range(query.range)?,
filter: store_usage_filter(query.filter),
cursor: query.cursor.map(store_cursor).transpose()?,
page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?,
page_size: ObservabilityPageSize::new(query.page_size.get())
.map_err(observability_error)?,
})
}
pub(crate) fn store_ops_error_filter(
filter: admin_observability::OpsErrorFilter,
) -> OpsErrorFilter {
OpsErrorFilter {
client_api_key_ref: filter.client_api_key_ref,
request_id: filter.request_id,
provider_account_ref: filter.provider_account_ref,
provider_kind: filter.provider_kind,
operation: filter.operation,
model: filter.model,
transport: filter.transport,
attempt_index: filter.attempt_index,
response_id: filter.response_id,
upstream_request_id: filter.upstream_request_id,
failure_kind: filter.failure_kind,
status_code: filter.status_code,
search: filter.search,
}
}
pub(crate) fn store_ops_error_query(
query: admin_observability::OpsErrorQuery,
) -> AdminStoreResult<OpsErrorQuery> {
Ok(OpsErrorQuery {
range: store_range(query.range)?,
filter: store_ops_error_filter(query.filter),
cursor: query.cursor.map(store_cursor).transpose()?,
page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?,
page_size: ObservabilityPageSize::new(query.page_size.get())
.map_err(observability_error)?,
})
}
pub(crate) const fn store_diagnostic_dimension(
dimension: admin_observability::DiagnosticDimension,
) -> DiagnosticDimension {
match dimension {
admin_observability::DiagnosticDimension::Provider => DiagnosticDimension::Provider,
admin_observability::DiagnosticDimension::Model => DiagnosticDimension::Model,
admin_observability::DiagnosticDimension::Account => DiagnosticDimension::Account,
admin_observability::DiagnosticDimension::ApiKey => DiagnosticDimension::ApiKey,
admin_observability::DiagnosticDimension::Transport => DiagnosticDimension::Transport,
admin_observability::DiagnosticDimension::Failure => DiagnosticDimension::Failure,
admin_observability::DiagnosticDimension::Status => DiagnosticDimension::Status,
}
}
pub(crate) fn admin_dashboard_observation(
observation: DashboardObservation,
) -> AdminStoreResult<admin_observability::DashboardObservation> {
let DashboardObservation {
range,
requests,
attempts,
provider_accounts,
trend,
account_usage,
recent_requests,
} = observation;
Ok(admin_observability::DashboardObservation {
range: admin_range(range),
requests: admin_request_metrics(requests)?,
attempts: admin_attempt_metrics(attempts)?,
provider_accounts: admin_account_pool_metrics(provider_accounts),
trend: trend
.into_iter()
.map(admin_request_metric_point)
.collect::<AdminStoreResult<_>>()?,
account_usage: account_usage
.into_iter()
.map(admin_dashboard_account_usage)
.collect::<AdminStoreResult<_>>()?,
recent_requests: recent_requests
.into_iter()
.map(admin_usage_record)
.collect::<AdminStoreResult<_>>()?,
})
}
pub(crate) const fn admin_range(range: ObservabilityRange) -> admin_observability::TimeRange {
admin_observability::TimeRange {
start: range.start,
end: range.end,
}
}
pub(crate) fn admin_request_metrics(
metrics: RequestMetrics,
) -> AdminStoreResult<admin_observability::RequestMetrics> {
Ok(admin_observability::RequestMetrics {
request_count: metrics.request_count,
success_count: metrics.success_count,
failure_count: metrics.failure_count,
cancelled_count: metrics.cancelled_count,
incomplete_count: metrics.incomplete_count,
caller_error_count: metrics.caller_error_count,
input_tokens: metrics.input_tokens,
output_tokens: metrics.output_tokens,
cached_tokens: metrics.cached_tokens,
cache_write_tokens: metrics.cache_write_tokens,
reasoning_tokens: metrics.reasoning_tokens,
total_tokens: metrics.total_tokens,
first_token_latency_sum_ms: metrics.first_token_latency_sum,
first_token_latency_count: metrics.first_token_latency_count,
latency_sum_ms: metrics.latency_sum,
latency_count: metrics.latency_count,
min_latency_ms: metrics.min_latency_ms,
max_latency_ms: metrics.max_latency_ms,
latency_percentiles: admin_latency_percentiles(metrics.latency_percentiles)?,
first_token_latency_percentiles: admin_latency_percentiles(
metrics.first_token_latency_percentiles,
)?,
cache_eligible_request_count: metrics.cache_eligible_request_count,
cache_hit_request_count: metrics.cache_hit_request_count,
})
}
pub(crate) fn admin_latency_percentiles(
percentiles: LatencyPercentiles,
) -> AdminStoreResult<admin_observability::LatencyPercentiles> {
Ok(admin_observability::LatencyPercentiles {
p50_ms: percentiles
.p50_ms
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
.transpose()
.map_err(|_| invalid_admin_percentile())?,
p95_ms: percentiles
.p95_ms
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
.transpose()
.map_err(|_| invalid_admin_percentile())?,
p99_ms: percentiles
.p99_ms
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
.transpose()
.map_err(|_| invalid_admin_percentile())?,
})
}
pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError {
observability_error(StoreError::InvalidData {
entity: "observability latency percentile",
message: "latency percentile is outside the admin contract".to_owned(),
})
}
pub(crate) fn admin_attempt_metrics(
metrics: AttemptMetrics,
) -> AdminStoreResult<admin_observability::AttemptMetrics> {
Ok(admin_observability::AttemptMetrics {
attempt_count: metrics.attempt_count,
success_count: metrics.success_count,
failure_count: metrics.failure_count,
cancelled_count: metrics.cancelled_count,
incomplete_count: metrics.incomplete_count,
rate_limited_count: metrics.rate_limited_count,
auth_failure_count: metrics.auth_failure_count,
provider_5xx_count: metrics.provider_5xx_count,
cost_coverage: admin_cost_coverage(metrics.cost_coverage),
costs: admin_currency_costs(metrics.costs)?,
})
}
pub(crate) const fn admin_cost_coverage(
coverage: CostCoverage,
) -> admin_observability::CostCoverage {
admin_observability::CostCoverage {
provider_reported_count: coverage.provider_reported_count,
calculated_count: coverage.calculated_count,
partial_count: 0,
unavailable_count: coverage.unavailable_count,
not_billable_count: 0,
}
}
pub(crate) fn admin_currency_costs(
costs: Vec<CurrencyCostTotal>,
) -> AdminStoreResult<Vec<admin_observability::CurrencyCost>> {
costs
.into_iter()
.map(|cost| {
Ok(admin_observability::CurrencyCost {
currency: cost.currency,
amount: admin_decimal_amount(cost.amount)?,
})
})
.collect()
}
pub(crate) fn admin_decimal_amount(
amount: DecimalAmount,
) -> AdminStoreResult<admin_observability::DecimalAmount> {
admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| {
observability_error(StoreError::InvalidData {
entity: "observability decimal amount",
message: "amount is outside the admin numeric contract".to_owned(),
})
})
}
pub(crate) fn admin_optional_decimal_amount(
amount: Option<DecimalAmount>,
) -> AdminStoreResult<Option<admin_observability::DecimalAmount>> {
amount.map(admin_decimal_amount).transpose()
}
pub(crate) const fn admin_account_pool_metrics(
metrics: ProviderAccountMetrics,
) -> admin_observability::AccountPoolMetrics {
admin_observability::AccountPoolMetrics {
total: metrics.total,
enabled: metrics.enabled,
unavailable: metrics.unavailable,
active: metrics.active,
rate_limited: metrics.rate_limited,
expired: metrics.expired,
invalid: metrics.invalid,
quota_exhausted: metrics.quota_exhausted,
refreshing: None,
disabled: metrics.disabled,
banned: metrics.banned,
}
}
pub(crate) fn admin_dashboard_account_usage(
usage: ProviderAccountUsageObservation,
) -> AdminStoreResult<admin_observability::DashboardAccountUsage> {
Ok(admin_observability::DashboardAccountUsage {
account_id: usage.account_id,
provider_kind: usage.provider_kind,
authentication_kind: usage.authentication_kind,
name: usage.name,
email: usage.email,
plan_type: usage.plan_type,
enabled: usage.enabled,
availability: usage.availability,
request_count: usage.request_count,
success_count: usage.success_count,
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
cached_tokens: usage.cached_tokens,
cache_write_tokens: usage.cache_write_tokens,
reasoning_tokens: usage.reasoning_tokens,
image_input_tokens: usage.image_input_tokens,
image_output_tokens: usage.image_output_tokens,
image_request_count: usage.image_request_count,
image_request_failed_count: usage.image_request_failed_count,
total_tokens: usage.total_tokens,
cost_coverage: admin_cost_coverage(usage.cost_coverage),
costs: admin_currency_costs(usage.costs)?,
last_used_at: usage.last_used_at,
request_buckets: usage
.request_buckets
.into_iter()
.map(
|bucket| admin_observability::DashboardAccountRequestBucket {
bucket_start: bucket.bucket_start,
request_count: bucket.request_count,
},
)
.collect(),
quota_used_percent: None,
models: usage
.models
.into_iter()
.map(admin_dashboard_account_model_usage)
.collect::<AdminStoreResult<_>>()?,
})
}
pub(crate) fn admin_dashboard_account_model_usage(
usage: ProviderAccountModelUsageObservation,
) -> AdminStoreResult<admin_observability::DashboardAccountModelUsage> {
Ok(admin_observability::DashboardAccountModelUsage {
model: usage.model,
request_count: usage.request_count,
success_count: usage.success_count,
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
cached_tokens: usage.cached_tokens,
cache_write_tokens: usage.cache_write_tokens,
reasoning_tokens: usage.reasoning_tokens,
image_input_tokens: usage.image_input_tokens,
image_output_tokens: usage.image_output_tokens,
image_request_count: usage.image_request_count,
image_request_failed_count: usage.image_request_failed_count,
total_tokens: usage.total_tokens,
cost_coverage: admin_cost_coverage(usage.cost_coverage),
costs: admin_currency_costs(usage.costs)?,
last_used_at: usage.last_used_at,
})
}
pub(crate) fn admin_request_metric_point(
point: RequestMetricPoint,
) -> AdminStoreResult<admin_observability::RequestMetricPoint> {
Ok(admin_observability::RequestMetricPoint {
bucket_start: point.bucket_start,
granularity: admin_granularity(point.granularity),
metrics: admin_request_metrics(point.metrics)?,
cost_coverage: admin_cost_coverage(point.cost_coverage),
costs: admin_currency_costs(point.costs)?,
})
}
pub(crate) fn admin_calculated_usage_billing_fact(
fact: CalculatedUsageBillingFact,
) -> AdminStoreResult<admin_observability::UsageCalculatedBillingFact> {
Ok(admin_observability::UsageCalculatedBillingFact {
bucket_start: fact.bucket_start,
provider_kind: fact.provider_kind,
upstream_model_id: fact.upstream_model_id,
service_tier: fact.service_tier,
input_tokens: fact.input_tokens,
output_tokens: fact.output_tokens,
cached_tokens: fact.cached_tokens,
cache_write_tokens: fact.cache_write_tokens,
total: admin_observability::CurrencyCost {
currency: fact.total.currency,
amount: admin_decimal_amount(fact.total.amount)?,
},
})
}
pub(crate) const fn admin_granularity(
granularity: ObservationGranularity,
) -> admin_observability::Granularity {
match granularity {
ObservationGranularity::FifteenMinutes => admin_observability::Granularity::FifteenMinutes,
ObservationGranularity::Hour => admin_observability::Granularity::Hour,
ObservationGranularity::Day => admin_observability::Granularity::Day,
}
}
pub(crate) fn admin_usage_page(
page: UsageRecordPage,
) -> AdminStoreResult<admin_observability::UsagePage> {
Ok(admin_observability::UsagePage {
items: page
.items
.into_iter()
.map(admin_usage_record)
.collect::<AdminStoreResult<_>>()?,
total: page.total,
next_cursor: page.next_cursor.map(admin_cursor),
})
}
pub(crate) fn admin_cursor(
cursor: ObservabilityCursor,
) -> admin_observability::ObservabilityCursor {
admin_observability::ObservabilityCursor {
observed_at: cursor.observed_at,
stable_id: cursor.stable_id,
}
}
pub(crate) fn admin_usage_record(
record: UsageRecord,
) -> AdminStoreResult<admin_observability::UsageRecord> {
let billing = match (&record.cost_amount, &record.cost_currency) {
(Some(amount), Some(currency)) => Some(admin_observability::UsageBilling::Total {
source: record.cost_source.clone(),
total: admin_observability::CurrencyCost {
currency: currency.clone(),
amount: admin_decimal_amount(amount.clone())?,
},
}),
(None, None) => None,
_ => {
return Err(observability_error(StoreError::InvalidData {
entity: "observability request billing",
message: "cost amount and currency must be present together".to_owned(),
}));
}
};
Ok(admin_observability::UsageRecord {
id: record.id,
client_api_key_ref: record.client_api_key_ref,
config_revision: record.config_revision,
protocol: record.protocol,
operation: record.operation,
endpoint: record.endpoint,
client_transport: record.client_transport,
requested_model_id: record.requested_model_id,
provider_kind: record.provider_kind,
provider_account_ref: record.provider_account_ref,
provider_account_name: record.provider_account_name,
provider_account_email: record.provider_account_email,
provider_account_authentication_kind: record.provider_account_authentication_kind,
upstream_model_id: record.upstream_model_id,
upstream_transport: record.upstream_transport,
http_version: record.http_version,
websocket_pool: record.websocket_pool,
service_tier: record.service_tier,
provider_metadata_json: record.provider_metadata_json,
attempt_count: record.attempt_count,
upstream_send_state: record.upstream_send_state,
downstream_committed_at: record.downstream_committed_at,
outcome: admin_request_outcome(&record.outcome)?,
client_status_code: record.client_status_code,
upstream_status_code: record.upstream_status_code,
client_response_id: record.client_response_id,
upstream_request_id: record.upstream_request_id,
upstream_response_id: record.upstream_response_id,
error_kind: record.error_kind,
provider_error_code: record.provider_error_code,
error_message: record.error_message,
retry_after_ms: record.retry_after_ms,
input_tokens: record.input_tokens,
output_tokens: record.output_tokens,
cached_tokens: record.cached_tokens,
cache_write_tokens: record.cache_write_tokens,
reasoning_tokens: record.reasoning_tokens,
image_input_tokens: record.image_input_tokens,
image_output_tokens: record.image_output_tokens,
total_tokens: record.total_tokens,
cost_source: record.cost_source,
cost_amount: admin_optional_decimal_amount(record.cost_amount)?,
cost_currency: record.cost_currency,
billing,
transport_decision_wait_ms: record.transport_decision_wait_ms,
connect_ms: record.connect_ms,
headers_ms: record.headers_ms,
first_event_ms: record.first_event_ms,
first_reasoning_ms: record.first_reasoning_ms,
first_text_ms: record.first_text_ms,
first_token_ms: record.first_token_ms,
provider_processing_ms: record.provider_processing_ms,
latency_ms: record.latency_ms,
client_ip: record.client_ip,
user_agent: record.user_agent,
reasoning_effort: record.reasoning_effort,
reasoning_preset: record.reasoning_preset,
request_kind: record.request_kind,
subagent_kind: record.subagent_kind,
compact: record.compact,
image_generation_requested: record.image_generation_requested,
image_generation_succeeded: record.image_generation_succeeded,
started_at: record.started_at,
deadline_at: record.deadline_at,
completed_at: record.completed_at,
})
}
pub(crate) fn admin_request_outcome(
outcome: &str,
) -> AdminStoreResult<admin_observability::RequestOutcome> {
admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| {
observability_error(StoreError::InvalidData {
entity: "observability request outcome",
message: "invalid request outcome".to_owned(),
})
})
}
pub(crate) fn admin_usage_detail(
detail: UsageRecordDetail,
) -> AdminStoreResult<admin_observability::UsageDetail> {
Ok(admin_observability::UsageDetail {
request: admin_usage_record(detail.request)?,
attempts: detail
.attempts
.into_iter()
.map(admin_usage_attempt)
.collect::<AdminStoreResult<_>>()?,
})
}
pub(crate) fn admin_usage_attempt(
attempt: UsageAttemptObservation,
) -> AdminStoreResult<admin_observability::UsageAttempt> {
Ok(admin_observability::UsageAttempt {
source: attempt.source,
id: attempt.id,
attempt_index: attempt.attempt_index,
component: attempt.component,
operation: attempt.operation,
provider_kind: attempt.provider_kind,
provider_account_ref: attempt.provider_account_ref,
provider_account_name: attempt.provider_account_name,
provider_account_email: attempt.provider_account_email,
provider_account_authentication_kind: attempt.provider_account_authentication_kind,
upstream_model_id: attempt.upstream_model_id,
upstream_transport: attempt.upstream_transport,
upstream_send_state: attempt.upstream_send_state,
outcome: admin_request_outcome(&attempt.outcome)?,
downstream_committed: attempt.downstream_committed,
status_code: attempt.status_code,
provider_error_code: attempt.provider_error_code,
failure_kind: attempt.failure_kind,
retry_after_ms: attempt.retry_after_ms,
upstream_request_id: attempt.upstream_request_id,
latency_ms: attempt.latency_ms,
message: attempt.message,
input_tokens: attempt.input_tokens,
output_tokens: attempt.output_tokens,
cached_tokens: attempt.cached_tokens,
cache_write_tokens: attempt.cache_write_tokens,
reasoning_tokens: attempt.reasoning_tokens,
total_tokens: attempt.total_tokens,
cost_source: attempt.cost_source,
cost_amount: admin_optional_decimal_amount(attempt.cost_amount)?,
cost_currency: attempt.cost_currency,
occurred_at: attempt.occurred_at,
})
}
pub(crate) fn admin_usage_overview(
overview: UsageOverview,
) -> AdminStoreResult<admin_observability::UsageOverview> {
Ok(admin_observability::UsageOverview {
range: admin_range(overview.range),
requests: admin_request_metrics(overview.requests)?,
attempts: admin_attempt_metrics(overview.attempts)?,
providers: overview
.providers
.into_iter()
.map(admin_provider_observation)
.collect(),
})
}
pub(crate) fn admin_provider_observation(
observation: ProviderObservation,
) -> admin_observability::ProviderObservation {
admin_observability::ProviderObservation {
provider_kind: observation.provider_kind,
request_count: observation.request_count,
attempt_count: observation.attempt_count,
failure_count: observation.failure_count,
total_tokens: observation.total_tokens,
}
}
pub(crate) fn admin_diagnostic_observation(
observation: DiagnosticObservation,
) -> AdminStoreResult<admin_observability::DiagnosticObservation> {
Ok(admin_observability::DiagnosticObservation {
key: observation.key,
name: observation.name,
request_count: observation.request_count,
success_count: observation.success_count,
failure_count: observation.failure_count,
attempt_count: observation.attempt_count,
total_tokens: observation.total_tokens,
average_latency_ms: observation.average_latency_ms,
latency_p95_ms: observation.latency_p95_ms,
cost_coverage: admin_cost_coverage(observation.cost_coverage),
costs: admin_currency_costs(observation.costs)?,
})
}
pub(crate) fn admin_ops_error_page(
page: OpsErrorPage,
) -> AdminStoreResult<admin_observability::OpsErrorPage> {
Ok(admin_observability::OpsErrorPage {
items: page.items.into_iter().map(admin_ops_error).collect(),
total: page.total,
next_cursor: page.next_cursor.map(admin_cursor),
})
}
pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError {
admin_observability::OpsError {
source: error.source,
event_id: error.event_id,
request_id: error.request_id,
attempt_index: error.attempt_index,
client_api_key_ref: error.client_api_key_ref,
component: error.component,
operation: error.operation,
endpoint: error.endpoint,
provider_kind: error.provider_kind,
provider_account_ref: error.provider_account_ref,
provider_account_name: error.provider_account_name,
provider_account_email: error.provider_account_email,
provider_account_authentication_kind: error.provider_account_authentication_kind,
upstream_model_id: error.upstream_model_id,
upstream_transport: error.upstream_transport,
failure_kind: error.failure_kind,
client_status_code: error.client_status_code,
upstream_status_code: error.upstream_status_code,
provider_error_code: error.provider_error_code,
client_response_id: error.client_response_id,
upstream_request_id: error.upstream_request_id,
latency_ms: error.latency_ms,
message: error.message,
occurrence_count: error.occurrence_count,
occurred_at: error.occurred_at,
stable_sort_id: error.stable_sort_id,
}
}
pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<UsageRecord> {
Ok(UsageRecord {
id: get(row, "id")?,
client_api_key_ref: get(row, "client_api_key_ref")?,
config_revision: unsigned(row, "config_revision")?,
protocol: get(row, "protocol")?,
operation: get(row, "operation")?,
endpoint: get(row, "endpoint")?,
client_transport: get(row, "client_transport")?,
requested_model_id: get(row, "requested_model_id")?,
provider_kind: get(row, "provider_kind")?,
provider_account_ref: get(row, "provider_account_ref")?,
provider_account_name: get(row, "provider_account_name")?,
provider_account_email: get(row, "provider_account_email")?,
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
upstream_model_id: get(row, "upstream_model_id")?,
upstream_transport: get(row, "upstream_transport")?,
http_version: get(row, "http_version")?,
websocket_pool: get(row, "websocket_pool")?,
service_tier: get(row, "service_tier")?,
provider_metadata_json: get::<Option<serde_json::Value>>(row, "provider_observation_json")?
.map(|value| serde_json::to_string(&value))
.transpose()
.map_err(|_| postgres_unavailable("encode provider observation"))?,
attempt_count: to_u32(get(row, "attempt_count")?)?,
upstream_send_state: get(row, "upstream_send_state")?,
downstream_committed_at: get(row, "downstream_committed_at")?,
outcome: get(row, "outcome")?,
client_status_code: optional_status(row, "client_status_code")?,
upstream_status_code: optional_status(row, "upstream_status_code")?,
client_response_id: opaque_response_id(row, "client_response_id")?,
upstream_request_id: get(row, "upstream_request_id")?,
upstream_response_id: opaque_response_id(row, "upstream_response_id")?,
error_kind: get(row, "error_kind")?,
provider_error_code: get(row, "provider_error_code")?,
error_message: get(row, "error_message")?,
retry_after_ms: optional_unsigned(row, "retry_after_ms")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
output_tokens: optional_unsigned(row, "output_tokens")?,
cached_tokens: optional_unsigned(row, "cached_tokens")?,
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
total_tokens: optional_unsigned(row, "total_tokens")?,
cost_source: get(row, "cost_source")?,
cost_amount: optional_decimal(row, "cost_amount")?,
cost_currency: get(row, "cost_currency")?,
transport_decision_wait_ms: optional_unsigned(row, "transport_decision_wait_ms")?,
connect_ms: optional_unsigned(row, "connect_ms")?,
headers_ms: optional_unsigned(row, "headers_ms")?,
first_event_ms: optional_unsigned(row, "first_event_ms")?,
first_reasoning_ms: optional_unsigned(row, "first_reasoning_ms")?,
first_text_ms: optional_unsigned(row, "first_text_ms")?,
first_token_ms: optional_unsigned(row, "first_token_ms")?,
provider_processing_ms: optional_unsigned(row, "provider_processing_ms")?,
latency_ms: optional_unsigned(row, "latency_ms")?,
client_ip: get(row, "client_ip")?,
user_agent: get(row, "user_agent")?,
reasoning_effort: get(row, "reasoning_effort")?,
reasoning_preset: get(row, "reasoning_preset")?,
request_kind: get(row, "request_kind")?,
subagent_kind: get(row, "subagent_kind")?,
compact: get(row, "compact")?,
image_generation_requested: get(row, "image_generation_requested")?,
image_generation_succeeded: get(row, "image_generation_succeeded")?,
started_at: get(row, "started_at")?,
deadline_at: get(row, "deadline_at")?,
completed_at: get(row, "completed_at")?,
})
}
pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<OpsErrorRecord> {
Ok(OpsErrorRecord {
source: get(row, "source")?,
event_id: get(row, "event_id")?,
request_id: get(row, "request_id")?,
attempt_index: get::<Option<i32>>(row, "attempt_index")?
.map(to_u32)
.transpose()?,
client_api_key_ref: get(row, "client_api_key_ref")?,
component: get(row, "component")?,
operation: get(row, "operation")?,
endpoint: get(row, "endpoint")?,
provider_kind: get(row, "provider_kind")?,
provider_account_ref: get(row, "provider_account_ref")?,
provider_account_name: get(row, "provider_account_name")?,
provider_account_email: get(row, "provider_account_email")?,
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
upstream_model_id: get(row, "upstream_model_id")?,
upstream_transport: get(row, "upstream_transport")?,
failure_kind: get(row, "failure_kind")?,
client_status_code: optional_status(row, "client_status_code")?,
upstream_status_code: optional_status(row, "upstream_status_code")?,
provider_error_code: get(row, "provider_error_code")?,
client_response_id: opaque_response_id(row, "client_response_id")?,
upstream_request_id: get(row, "upstream_request_id")?,
latency_ms: optional_unsigned(row, "latency_ms")?,
message: get(row, "message")?,
occurrence_count: to_u32(get(row, "occurrence_count")?)?,
occurred_at: get(row, "occurred_at")?,
stable_sort_id: get(row, "stable_sort_id")?,
})
}
pub(crate) fn cost_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CurrencyCostTotal> {
Ok(CurrencyCostTotal {
currency: get(row, "cost_currency")?,
amount: DecimalAmount::from_str(&get::<String>(row, "amount")?)?,
})
}
pub(crate) fn calculated_usage_billing_fact_from_row(
row: &sqlx::postgres::PgRow,
) -> StoreResult<CalculatedUsageBillingFact> {
Ok(CalculatedUsageBillingFact {
bucket_start: get(row, "bucket_start")?,
provider_kind: get(row, "provider_kind")?,
upstream_model_id: get(row, "upstream_model_id")?,
service_tier: get(row, "service_tier")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
output_tokens: optional_unsigned(row, "output_tokens")?,
cached_tokens: optional_unsigned(row, "cached_tokens")?,
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
total: cost_from_row(row)?,
})
}
pub(crate) fn optional_decimal(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> StoreResult<Option<DecimalAmount>> {
get::<Option<String>>(row, column)?
.map(|value| DecimalAmount::from_str(&value))
.transpose()
}
pub(crate) fn opaque_response_id(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> StoreResult<Option<String>> {
get::<Option<Vec<u8>>>(row, column)?
.map(String::from_utf8)
.transpose()
.map_err(|_| invalid(column))
}
pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> {
let mut unique = BTreeSet::new();
for account_id in account_ids {
validate_text(account_id, MAX_FILTER_BYTES, "provider account ID")?;
if !unique.insert(account_id.as_str()) {
return Err(invalid("account usage query contains duplicate IDs"));
}
}
Ok(())
}
pub(crate) fn validate_optional_text(
value: Option<&str>,
max_bytes: usize,
field: &'static str,
) -> StoreResult<()> {
value.map_or(Ok(()), |value| validate_text(value, max_bytes, field))
}
pub(crate) fn validate_text(value: &str, max_bytes: usize, field: &'static str) -> StoreResult<()> {
if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) {
return Err(invalid(&format!("{field} is invalid")));
}
Ok(())
}
pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult<T>
where
T: sqlx::Decode<'r, Postgres> + sqlx::Type<Postgres>,
{
row.try_get(column).map_err(|_| invalid(column))
}
pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult<u64> {
to_u64(get(row, column)?)
}
pub(crate) fn optional_unsigned(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> StoreResult<Option<u64>> {
get::<Option<i64>>(row, column)?.map(to_u64).transpose()
}
pub(crate) fn optional_status(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> StoreResult<Option<u16>> {
get::<Option<i32>>(row, column)?
.map(|value| {
u16::try_from(value)
.ok()
.filter(|value| (100..=599).contains(value))
.ok_or_else(|| invalid("status code is outside its supported range"))
})
.transpose()
}
pub(crate) fn to_u64(value: i64) -> StoreResult<u64> {
u64::try_from(value).map_err(|_| invalid("numeric observation is negative"))
}
pub(crate) fn to_u32(value: i32) -> StoreResult<u32> {
u32::try_from(value).map_err(|_| invalid("integer observation is negative"))
}
pub(crate) fn invalid(message: &str) -> StoreError {
StoreError::InvalidData {
entity: "observability",
message: message.to_owned(),
}
}
@@ -0,0 +1,29 @@
//! 从 `model_requests`、`ops_events` 与账号公共投影读取观测事实。
use std::{
collections::{BTreeMap, BTreeSet, HashMap},
str::FromStr,
};
use async_trait::async_trait;
use chrono::{DateTime, TimeDelta, Utc};
use gateway_admin::{
model::observability as admin_observability,
ports::store::{AdminStoreResult, ObservabilityStore as AdminObservabilityStore},
};
use sqlx::{PgPool, Postgres, QueryBuilder, Row};
use crate::{
DecimalAmount, StoreError, StoreResult, admin_store_error, postgres_unavailable,
require_nonempty,
};
mod admin_adapter;
mod mapping;
mod model;
mod queries;
pub use admin_adapter::*;
pub(crate) use mapping::*;
pub use model::*;
pub(crate) use queries::*;
@@ -0,0 +1,715 @@
//! 查询模型、校验与观测端口契约。
use super::*;
pub(crate) const MAX_PAGE_SIZE: u16 = 100;
pub(crate) const MAX_FILTER_BYTES: usize = 256;
pub(crate) const MAX_SEARCH_BYTES: usize = 512;
pub(crate) const MAX_ACCOUNT_IDS: usize = 200;
/// 概览卡只展示最近使用的四个账号;完整账号用量由账号管理页单独查询。
pub(crate) const DASHBOARD_ACCOUNT_LIMIT: u16 = 4;
pub(crate) const DIAGNOSTIC_LIMIT: i64 = 100;
pub(crate) const ACCOUNT_USAGE_TIMELINE_HOURS: i64 = 24;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ObservabilityRange {
pub start: DateTime<Utc>,
pub end: DateTime<Utc>,
}
impl ObservabilityRange {
pub fn new(start: DateTime<Utc>, end: DateTime<Utc>) -> StoreResult<Self> {
if end.signed_duration_since(start) <= TimeDelta::zero() {
return Err(invalid("time range must be positive"));
}
Ok(Self { start, end })
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ObservabilityPageSize(u16);
impl ObservabilityPageSize {
pub fn new(value: u16) -> StoreResult<Self> {
if value == 0 || value > MAX_PAGE_SIZE {
return Err(invalid("page size must be between 1 and 100"));
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u16 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ObservabilityPageNumber(u32);
impl ObservabilityPageNumber {
pub fn new(value: u32) -> StoreResult<Self> {
if value == 0 {
return Err(invalid("page number must be positive"));
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u32 {
self.0
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ObservabilityCursor {
pub observed_at: DateTime<Utc>,
pub stable_id: String,
}
impl ObservabilityCursor {
pub fn new(observed_at: DateTime<Utc>, stable_id: impl Into<String>) -> StoreResult<Self> {
let stable_id = stable_id.into();
validate_text(&stable_id, MAX_FILTER_BYTES, "cursor ID")?;
Ok(Self {
observed_at,
stable_id,
})
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UsageRecordFilter {
pub client_api_key_ref: Option<String>,
pub request_id: Option<String>,
pub provider_account_ref: Option<String>,
pub operation: Option<String>,
pub provider_kind: Option<String>,
pub model: Option<String>,
pub outcome: Option<String>,
pub status_code: Option<u16>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub search: Option<String>,
}
impl UsageRecordFilter {
pub fn validate(&self) -> StoreResult<()> {
for (value, field) in [
(self.client_api_key_ref.as_deref(), "client API key filter"),
(self.request_id.as_deref(), "request ID filter"),
(
self.provider_account_ref.as_deref(),
"provider account filter",
),
(self.operation.as_deref(), "operation filter"),
(self.provider_kind.as_deref(), "provider filter"),
(self.model.as_deref(), "model filter"),
(self.outcome.as_deref(), "outcome filter"),
(self.transport.as_deref(), "transport filter"),
(
self.upstream_request_id.as_deref(),
"upstream request ID filter",
),
] {
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
}
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
if self
.status_code
.is_some_and(|status| !(100..=599).contains(&status))
{
return Err(invalid("status code filter must be between 100 and 599"));
}
if self
.attempt_index
.is_some_and(|index| index == 0 || i32::try_from(index).is_err())
{
return Err(invalid("attempt index filter is out of range"));
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageRecordQuery {
pub range: ObservabilityRange,
pub filter: UsageRecordFilter,
pub cursor: Option<ObservabilityCursor>,
pub page: ObservabilityPageNumber,
pub page_size: ObservabilityPageSize,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OpsErrorFilter {
pub client_api_key_ref: Option<String>,
pub request_id: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_kind: Option<String>,
pub operation: Option<String>,
pub model: Option<String>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub failure_kind: Option<String>,
pub status_code: Option<u16>,
pub search: Option<String>,
}
impl OpsErrorFilter {
pub fn validate(&self) -> StoreResult<()> {
for (value, field) in [
(self.client_api_key_ref.as_deref(), "client API key filter"),
(self.request_id.as_deref(), "request ID filter"),
(
self.provider_account_ref.as_deref(),
"provider account filter",
),
(self.provider_kind.as_deref(), "provider filter"),
(self.operation.as_deref(), "operation filter"),
(self.model.as_deref(), "model filter"),
(self.transport.as_deref(), "transport filter"),
(
self.upstream_request_id.as_deref(),
"upstream request ID filter",
),
(self.failure_kind.as_deref(), "failure kind filter"),
] {
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
}
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
if self
.status_code
.is_some_and(|status| !(100..=599).contains(&status))
{
return Err(invalid("status code filter must be between 100 and 599"));
}
if self
.attempt_index
.is_some_and(|index| index == 0 || i32::try_from(index).is_err())
{
return Err(invalid("attempt index filter is out of range"));
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OpsErrorQuery {
pub range: ObservabilityRange,
pub filter: OpsErrorFilter,
pub cursor: Option<ObservabilityCursor>,
pub page: ObservabilityPageNumber,
pub page_size: ObservabilityPageSize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DiagnosticDimension {
Provider,
Model,
Account,
ApiKey,
Transport,
Failure,
Status,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CurrencyCostTotal {
pub currency: String,
pub amount: DecimalAmount,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CostCoverage {
pub provider_reported_count: u64,
pub calculated_count: u64,
pub unavailable_count: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RequestMetrics {
pub request_count: u64,
pub success_count: u64,
pub failure_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub caller_error_count: u64,
pub input_tokens: u64,
pub output_tokens: u64,
pub cached_tokens: u64,
pub cache_write_tokens: u64,
pub reasoning_tokens: u64,
pub total_tokens: u64,
pub first_token_latency_sum: u64,
pub first_token_latency_count: u64,
pub latency_sum: u64,
pub latency_count: u64,
pub max_latency_ms: Option<u64>,
pub min_latency_ms: Option<u64>,
/// 分母:`input_tokens is not null`,即上游确实报告过 input token 事实的请求。
pub cache_eligible_request_count: u64,
/// 分子:分母集合中 `cached_tokens > 0` 的请求。
pub cache_hit_request_count: u64,
pub latency_percentiles: LatencyPercentiles,
pub first_token_latency_percentiles: LatencyPercentiles,
}
impl RequestMetrics {
/// 请求级 cache hit rate;没有 input token 事实时返回 `None`。
#[must_use]
pub fn cache_hit_request_rate(&self) -> Option<f64> {
(self.cache_eligible_request_count > 0)
.then(|| self.cache_hit_request_count as f64 / self.cache_eligible_request_count as f64)
}
}
/// PostgreSQL `percentile_cont` 的非负、有限毫秒值;bits 保留插值小数且可安全比较。
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct PercentileMilliseconds(u64);
impl PercentileMilliseconds {
pub(crate) fn new(value: f64) -> StoreResult<Self> {
if !value.is_finite() || value < 0.0 {
return Err(postgres_unavailable("decode latency percentile"));
}
Ok(Self(value.to_bits()))
}
#[must_use]
pub const fn as_f64(self) -> f64 {
f64::from_bits(self.0)
}
}
impl std::fmt::Debug for PercentileMilliseconds {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.as_f64().fmt(formatter)
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct LatencyPercentiles {
pub p50_ms: Option<PercentileMilliseconds>,
pub p95_ms: Option<PercentileMilliseconds>,
pub p99_ms: Option<PercentileMilliseconds>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AttemptMetrics {
pub attempt_count: u64,
pub success_count: u64,
pub failure_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub rate_limited_count: u64,
pub auth_failure_count: u64,
pub provider_5xx_count: u64,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCostTotal>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ObservationGranularity {
FifteenMinutes,
Hour,
Day,
}
impl ObservationGranularity {
#[must_use]
pub const fn seconds(self) -> i64 {
match self {
Self::FifteenMinutes => 15 * 60,
Self::Hour => 60 * 60,
Self::Day => 24 * 60 * 60,
}
}
pub(crate) const fn sql_interval(self) -> &'static str {
match self {
Self::FifteenMinutes => "15 minutes",
Self::Hour => "1 hour",
Self::Day => "1 day",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestMetricPoint {
pub bucket_start: DateTime<Utc>,
pub granularity: ObservationGranularity,
pub metrics: RequestMetrics,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCostTotal>,
}
/// 已完整交付且由 Provider 计算费用的请求事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CalculatedUsageBillingFact {
pub bucket_start: DateTime<Utc>,
pub provider_kind: String,
pub upstream_model_id: String,
pub service_tier: Option<String>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub total: CurrencyCostTotal,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ProviderAccountMetrics {
pub total: u64,
pub enabled: u64,
pub unavailable: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub quota_exhausted: u64,
pub disabled: u64,
pub banned: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountRequestBucket {
pub bucket_start: DateTime<Utc>,
pub request_count: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountUsageObservation {
pub account_id: String,
pub provider_kind: String,
pub authentication_kind: String,
pub name: String,
pub email: Option<String>,
pub plan_type: Option<String>,
pub enabled: bool,
pub availability: String,
pub request_count: u64,
pub success_count: u64,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub image_input_tokens: Option<u64>,
pub image_output_tokens: Option<u64>,
pub image_request_count: u64,
pub image_request_failed_count: u64,
pub total_tokens: Option<u64>,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCostTotal>,
pub last_used_at: Option<DateTime<Utc>>,
pub request_buckets: Vec<ProviderAccountRequestBucket>,
pub models: Vec<ProviderAccountModelUsageObservation>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountModelUsageObservation {
pub model: String,
pub request_count: u64,
pub success_count: u64,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub image_input_tokens: Option<u64>,
pub image_output_tokens: Option<u64>,
pub image_request_count: u64,
pub image_request_failed_count: u64,
pub total_tokens: Option<u64>,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCostTotal>,
pub last_used_at: DateTime<Utc>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountUsageQuery {
pub range: ObservabilityRange,
pub account_ids: Option<Vec<String>>,
pub limit: u16,
pub(crate) include_hourly_request_buckets: bool,
}
impl ProviderAccountUsageQuery {
pub fn for_accounts(range: ObservabilityRange, account_ids: Vec<String>) -> StoreResult<Self> {
if account_ids.is_empty() || account_ids.len() > MAX_ACCOUNT_IDS {
return Err(invalid(
"account usage query requires between 1 and 200 IDs",
));
}
validate_account_ids(&account_ids)?;
Ok(Self {
range,
limit: u16::try_from(account_ids.len())
.map_err(|_| invalid("account usage query is too large"))?,
account_ids: Some(account_ids),
include_hourly_request_buckets: false,
})
}
pub fn recent(range: ObservabilityRange, limit: u16) -> StoreResult<Self> {
if limit == 0 || usize::from(limit) > MAX_ACCOUNT_IDS {
return Err(invalid("account usage limit must be between 1 and 200"));
}
Ok(Self {
range,
account_ids: None,
limit,
include_hourly_request_buckets: false,
})
}
pub fn with_hourly_request_buckets(mut self) -> StoreResult<Self> {
if self.range.end.signed_duration_since(self.range.start)
> TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS)
{
return Err(invalid("account request timeline cannot exceed 24 hours"));
}
self.include_hourly_request_buckets = true;
Ok(self)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DashboardObservation {
pub range: ObservabilityRange,
pub requests: RequestMetrics,
pub attempts: AttemptMetrics,
pub provider_accounts: ProviderAccountMetrics,
pub trend: Vec<RequestMetricPoint>,
pub account_usage: Vec<ProviderAccountUsageObservation>,
pub recent_requests: Vec<UsageRecord>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageRecord {
pub id: String,
pub client_api_key_ref: String,
pub config_revision: u64,
pub protocol: String,
pub operation: String,
pub endpoint: String,
pub client_transport: String,
pub requested_model_id: String,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
pub provider_account_email: Option<String>,
pub provider_account_authentication_kind: Option<String>,
pub upstream_model_id: Option<String>,
pub upstream_transport: Option<String>,
pub http_version: Option<String>,
pub websocket_pool: Option<String>,
pub service_tier: Option<String>,
pub provider_metadata_json: Option<String>,
pub attempt_count: u32,
pub upstream_send_state: String,
pub downstream_committed_at: Option<DateTime<Utc>>,
pub outcome: String,
pub client_status_code: Option<u16>,
pub upstream_status_code: Option<u16>,
pub client_response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub upstream_response_id: Option<String>,
pub error_kind: Option<String>,
pub provider_error_code: Option<String>,
pub error_message: Option<String>,
pub retry_after_ms: Option<u64>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub image_input_tokens: Option<u64>,
pub image_output_tokens: Option<u64>,
pub total_tokens: Option<u64>,
pub cost_source: String,
pub cost_amount: Option<DecimalAmount>,
pub cost_currency: Option<String>,
pub transport_decision_wait_ms: Option<u64>,
pub connect_ms: Option<u64>,
pub headers_ms: Option<u64>,
pub first_event_ms: Option<u64>,
pub first_reasoning_ms: Option<u64>,
pub first_text_ms: Option<u64>,
pub first_token_ms: Option<u64>,
pub provider_processing_ms: Option<u64>,
pub latency_ms: Option<u64>,
pub client_ip: Option<String>,
pub user_agent: Option<String>,
pub reasoning_effort: Option<String>,
pub reasoning_preset: Option<String>,
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub image_generation_requested: bool,
pub image_generation_succeeded: Option<bool>,
pub started_at: DateTime<Utc>,
pub deadline_at: DateTime<Utc>,
pub completed_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageRecordPage {
pub items: Vec<UsageRecord>,
pub total: u64,
pub next_cursor: Option<ObservabilityCursor>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageAttemptObservation {
pub source: String,
pub id: String,
pub attempt_index: u32,
pub component: String,
pub operation: String,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
pub provider_account_email: Option<String>,
pub provider_account_authentication_kind: Option<String>,
pub upstream_model_id: Option<String>,
pub upstream_transport: Option<String>,
pub upstream_send_state: Option<String>,
pub outcome: String,
pub downstream_committed: bool,
pub status_code: Option<u16>,
pub provider_error_code: Option<String>,
pub failure_kind: Option<String>,
pub retry_after_ms: Option<u64>,
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub message: Option<String>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cached_tokens: Option<u64>,
pub cache_write_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
pub total_tokens: Option<u64>,
pub cost_source: Option<String>,
pub cost_amount: Option<DecimalAmount>,
pub cost_currency: Option<String>,
pub occurred_at: DateTime<Utc>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageRecordDetail {
pub request: UsageRecord,
pub attempts: Vec<UsageAttemptObservation>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderObservation {
pub provider_kind: String,
pub request_count: u64,
pub attempt_count: u64,
pub failure_count: u64,
pub total_tokens: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageOverview {
pub range: ObservabilityRange,
pub requests: RequestMetrics,
pub attempts: AttemptMetrics,
pub providers: Vec<ProviderObservation>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DiagnosticObservation {
pub key: String,
pub name: String,
pub request_count: u64,
pub success_count: u64,
pub failure_count: u64,
pub attempt_count: u64,
pub total_tokens: u64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub cost_coverage: CostCoverage,
pub costs: Vec<CurrencyCostTotal>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OpsErrorRecord {
pub source: String,
pub event_id: String,
pub request_id: Option<String>,
pub attempt_index: Option<u32>,
pub client_api_key_ref: Option<String>,
pub component: String,
pub operation: String,
pub endpoint: Option<String>,
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub provider_account_name: Option<String>,
pub provider_account_email: Option<String>,
pub provider_account_authentication_kind: Option<String>,
pub upstream_model_id: Option<String>,
pub upstream_transport: Option<String>,
pub failure_kind: String,
pub client_status_code: Option<u16>,
pub upstream_status_code: Option<u16>,
pub provider_error_code: Option<String>,
pub client_response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub message: String,
pub occurrence_count: u32,
pub occurred_at: DateTime<Utc>,
pub stable_sort_id: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OpsErrorPage {
pub items: Vec<OpsErrorRecord>,
pub total: u64,
pub next_cursor: Option<ObservabilityCursor>,
}
#[async_trait]
pub trait ObservabilityRepository: Send + Sync {
async fn dashboard_summary(
&self,
range: ObservabilityRange,
) -> StoreResult<DashboardObservation>;
async fn dashboard_trend(
&self,
range: ObservabilityRange,
) -> StoreResult<Vec<RequestMetricPoint>>;
async fn usage_trend(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<Vec<RequestMetricPoint>>;
async fn usage_calculated_billing_facts(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<Vec<CalculatedUsageBillingFact>>;
async fn provider_account_usage(
&self,
query: ProviderAccountUsageQuery,
) -> StoreResult<Vec<ProviderAccountUsageObservation>>;
async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult<UsageRecordPage>;
async fn usage_record_detail(&self, request_id: &str) -> StoreResult<UsageRecordDetail>;
async fn usage_summary(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
) -> StoreResult<UsageOverview>;
async fn usage_diagnostics(
&self,
range: ObservabilityRange,
filter: UsageRecordFilter,
dimension: DiagnosticDimension,
) -> StoreResult<Vec<DiagnosticObservation>>;
async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult<OpsErrorPage>;
}
@@ -0,0 +1,336 @@
//! 账号用量查询族。
use super::super::*;
pub(crate) async fn provider_account_usage(
pool: &PgPool,
query: ProviderAccountUsageQuery,
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
if let Some(account_ids) = &query.account_ids {
validate_account_ids(account_ids)?;
}
if query.limit == 0 || usize::from(query.limit) > MAX_ACCOUNT_IDS {
return Err(invalid("account usage limit must be between 1 and 200"));
}
let mut statement = QueryBuilder::<Postgres>::new(
"select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
pa.plan_type, pa.enabled, pa.availability,
count(mr.id)::bigint as request_count,
count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count,
sum(mr.input_tokens)::bigint as input_tokens,
sum(mr.output_tokens)::bigint as output_tokens,
sum(mr.cached_tokens)::bigint as cached_tokens,
sum(mr.cache_write_tokens)::bigint as cache_write_tokens,
sum(mr.reasoning_tokens)::bigint as reasoning_tokens,
sum(mr.image_input_tokens)::bigint as image_input_tokens,
sum(mr.image_output_tokens)::bigint as image_output_tokens,
count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint
as image_request_count,
count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint
as image_request_failed_count,
sum(mr.total_tokens)::bigint as total_tokens,
count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint
as provider_reported_count,
count(mr.id) filter (where mr.cost_source = 'calculated')::bigint
as calculated_count,
count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint
as unavailable_count,
max(mr.started_at) as last_used_at
from provider_accounts pa
left join model_requests mr
on mr.provider_account_ref = pa.id and mr.started_at >= ",
);
statement.push_bind(query.range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(query.range.end);
push_completed_usage_fact_filter(&mut statement, "mr");
if let Some(account_ids) = &query.account_ids {
statement.push(" where pa.id = any(");
statement.push_bind(account_ids.clone());
statement.push("::text[])");
}
statement.push(
" group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
pa.plan_type, pa.enabled, pa.availability
order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ",
);
statement.push_bind(i64::from(query.limit));
let rows = statement
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account usage"))?;
let mut observations = rows
.iter()
.map(provider_account_from_row)
.collect::<StoreResult<Vec<_>>>()?;
if observations.is_empty() {
return Ok(observations);
}
let account_ids = observations
.iter()
.map(|item| item.account_id.clone())
.collect::<Vec<_>>();
let mut request_buckets = if query.include_hourly_request_buckets {
provider_account_request_buckets(pool, query.range, &account_ids).await?
} else {
HashMap::new()
};
let account_costs = account_costs(pool, query.range, &account_ids).await?;
let model_rows = account_model_rows(pool, query.range, &account_ids).await?;
let model_costs = account_model_costs(pool, query.range, &account_ids).await?;
let account_positions = observations
.iter()
.enumerate()
.map(|(index, item)| (item.account_id.clone(), index))
.collect::<HashMap<_, _>>();
for observation in &mut observations {
observation.costs = account_costs
.get(&observation.account_id)
.cloned()
.unwrap_or_default();
observation.request_buckets = request_buckets
.remove(&observation.account_id)
.unwrap_or_default();
}
for row in model_rows {
let account_id: String = get(&row, "provider_account_ref")?;
let Some(position) = account_positions.get(&account_id).copied() else {
continue;
};
let model: String = get(&row, "model")?;
observations[position]
.models
.push(ProviderAccountModelUsageObservation {
costs: model_costs
.get(&(account_id, model.clone()))
.cloned()
.unwrap_or_default(),
model,
request_count: unsigned(&row, "request_count")?,
success_count: unsigned(&row, "success_count")?,
input_tokens: optional_unsigned(&row, "input_tokens")?,
output_tokens: optional_unsigned(&row, "output_tokens")?,
cached_tokens: optional_unsigned(&row, "cached_tokens")?,
cache_write_tokens: optional_unsigned(&row, "cache_write_tokens")?,
reasoning_tokens: optional_unsigned(&row, "reasoning_tokens")?,
image_input_tokens: optional_unsigned(&row, "image_input_tokens")?,
image_output_tokens: optional_unsigned(&row, "image_output_tokens")?,
image_request_count: unsigned(&row, "image_request_count")?,
image_request_failed_count: unsigned(&row, "image_request_failed_count")?,
total_tokens: optional_unsigned(&row, "total_tokens")?,
cost_coverage: coverage_from_row(&row)?,
last_used_at: get(&row, "last_used_at")?,
});
}
Ok(observations)
}
pub(crate) async fn provider_account_request_buckets(
pool: &PgPool,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<HashMap<String, Vec<ProviderAccountRequestBucket>>> {
let rows = sqlx::query(
"select provider_account_ref,
floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index,
count(*)::bigint as request_count
from model_requests mr
where mr.provider_account_ref = any($2::text[])
and mr.started_at >= $1 and mr.started_at < $3
and mr.outcome = 'succeeded'
and mr.downstream_committed_at is not null
and mr.client_status_code between 200 and 399
group by mr.provider_account_ref, bucket_index
order by mr.provider_account_ref, bucket_index",
)
.bind(range.start)
.bind(account_ids)
.bind(range.end)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account request timeline"))?;
let mut observed = HashMap::<String, BTreeMap<u64, u64>>::new();
for row in rows {
observed
.entry(get(&row, "provider_account_ref")?)
.or_default()
.insert(
unsigned(&row, "bucket_index")?,
unsigned(&row, "request_count")?,
);
}
let step = TimeDelta::hours(1);
let mut timelines = HashMap::with_capacity(account_ids.len());
for account_id in account_ids {
let mut account_observed = observed.remove(account_id).unwrap_or_default();
let mut bucket_start = range.start;
let mut bucket_index = 0_u64;
let mut buckets = Vec::new();
while bucket_start < range.end {
buckets.push(ProviderAccountRequestBucket {
bucket_start,
request_count: account_observed.remove(&bucket_index).unwrap_or_default(),
});
bucket_start = bucket_start
.checked_add_signed(step)
.ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?;
bucket_index = bucket_index
.checked_add(1)
.ok_or_else(|| invalid("account request timeline exceeds supported buckets"))?;
}
timelines.insert(account_id.clone(), buckets);
}
Ok(timelines)
}
pub(crate) async fn account_costs(
pool: &PgPool,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<HashMap<String, Vec<CurrencyCostTotal>>> {
let rows = sqlx::query(
"select mr.provider_account_ref, mr.cost_currency, sum(mr.cost_amount)::text as amount
from model_requests mr
where mr.provider_account_ref = any($1::text[])
and mr.started_at >= $2 and mr.started_at < $3
and mr.outcome = 'succeeded'
and mr.downstream_committed_at is not null
and mr.client_status_code between 200 and 399
and mr.cost_amount is not null and mr.cost_currency is not null
group by mr.provider_account_ref, mr.cost_currency
order by mr.provider_account_ref, mr.cost_currency",
)
.bind(account_ids)
.bind(range.start)
.bind(range.end)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account costs"))?;
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
for row in rows {
costs
.entry(get(&row, "provider_account_ref")?)
.or_default()
.push(cost_from_row(&row)?);
}
Ok(costs)
}
pub(crate) async fn account_model_rows(
pool: &PgPool,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<Vec<sqlx::postgres::PgRow>> {
sqlx::query(
"select mr.provider_account_ref,
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
count(*)::bigint as request_count,
count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count,
sum(mr.input_tokens)::bigint as input_tokens,
sum(mr.output_tokens)::bigint as output_tokens,
sum(mr.cached_tokens)::bigint as cached_tokens,
sum(mr.cache_write_tokens)::bigint as cache_write_tokens,
sum(mr.reasoning_tokens)::bigint as reasoning_tokens,
sum(mr.image_input_tokens)::bigint as image_input_tokens,
sum(mr.image_output_tokens)::bigint as image_output_tokens,
count(*) filter (where mr.image_generation_succeeded is true)::bigint
as image_request_count,
count(*) filter (where mr.image_generation_succeeded is false)::bigint
as image_request_failed_count,
sum(mr.total_tokens)::bigint as total_tokens,
count(*) filter (where mr.cost_source = 'provider_reported')::bigint
as provider_reported_count,
count(*) filter (where mr.cost_source = 'calculated')::bigint
as calculated_count,
count(*) filter (where mr.cost_source = 'unavailable')::bigint
as unavailable_count,
max(mr.started_at) as last_used_at
from model_requests mr
where mr.provider_account_ref = any($1::text[])
and mr.started_at >= $2 and mr.started_at < $3
and mr.outcome = 'succeeded'
and mr.downstream_committed_at is not null
and mr.client_status_code between 200 and 399
group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id)
order by mr.provider_account_ref, request_count desc, model",
)
.bind(account_ids)
.bind(range.start)
.bind(range.end)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account model usage"))
}
pub(crate) async fn account_model_costs(
pool: &PgPool,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<HashMap<(String, String), Vec<CurrencyCostTotal>>> {
let rows = sqlx::query(
"select mr.provider_account_ref,
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
mr.cost_currency, sum(mr.cost_amount)::text as amount
from model_requests mr
where mr.provider_account_ref = any($1::text[])
and mr.started_at >= $2 and mr.started_at < $3
and mr.outcome = 'succeeded'
and mr.downstream_committed_at is not null
and mr.client_status_code between 200 and 399
and mr.cost_amount is not null and mr.cost_currency is not null
group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id),
mr.cost_currency
order by mr.provider_account_ref, model, mr.cost_currency",
)
.bind(account_ids)
.bind(range.start)
.bind(range.end)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account model costs"))?;
let mut costs = HashMap::<(String, String), Vec<CurrencyCostTotal>>::new();
for row in rows {
costs
.entry((get(&row, "provider_account_ref")?, get(&row, "model")?))
.or_default()
.push(cost_from_row(&row)?);
}
Ok(costs)
}
pub(crate) fn provider_account_from_row(
row: &sqlx::postgres::PgRow,
) -> StoreResult<ProviderAccountUsageObservation> {
Ok(ProviderAccountUsageObservation {
account_id: get(row, "id")?,
provider_kind: get(row, "provider_kind")?,
authentication_kind: get(row, "authentication_kind")?,
name: get(row, "name")?,
email: get(row, "email")?,
plan_type: get(row, "plan_type")?,
enabled: get(row, "enabled")?,
availability: get(row, "availability")?,
request_count: unsigned(row, "request_count")?,
success_count: unsigned(row, "success_count")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
output_tokens: optional_unsigned(row, "output_tokens")?,
cached_tokens: optional_unsigned(row, "cached_tokens")?,
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
image_request_count: unsigned(row, "image_request_count")?,
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
total_tokens: optional_unsigned(row, "total_tokens")?,
cost_coverage: coverage_from_row(row)?,
costs: Vec::new(),
last_used_at: get(row, "last_used_at")?,
request_buckets: Vec::new(),
models: Vec::new(),
})
}
@@ -0,0 +1,553 @@
//! Dashboard 与趋势查询族。
use super::super::*;
pub(crate) async fn request_metrics(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<RequestMetrics> {
filter.validate()?;
// 结果计数覆盖范围内全部请求(成功率/失败率的分母分子);用量、缓存、
// 延迟与成本聚合仅统计用量事实(已完整交付客户端的成功响应)。
let fact = completed_usage_fact_predicate("mr");
let mut query = QueryBuilder::<Postgres>::new(format!(
"select count(*)::bigint as request_count,
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
count(*) filter (where outcome = 'failed')::bigint as failure_count,
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
count(*) filter (where client_status_code between 400 and 499)::bigint
as caller_error_count,
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
as cache_write_tokens,
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
as reasoning_tokens,
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
as first_token_latency_sum,
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
count(latency_ms) filter (where {fact})::bigint as latency_count,
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
count(*) filter (where {fact} and input_tokens is not null)::bigint
as cache_eligible_request_count,
count(*) filter (where {fact} and input_tokens is not null
and cached_tokens > 0)::bigint
as cache_hit_request_count,
percentile_cont(0.50) within group (order by latency_ms)
filter (where {fact}) as latency_p50_ms,
percentile_cont(0.95) within group (order by latency_ms)
filter (where {fact}) as latency_p95_ms,
percentile_cont(0.99) within group (order by latency_ms)
filter (where {fact}) as latency_p99_ms,
percentile_cont(0.50) within group (order by first_token_ms)
filter (where {fact}) as first_token_p50_ms,
percentile_cont(0.95) within group (order by first_token_ms)
filter (where {fact}) as first_token_p95_ms,
percentile_cont(0.99) within group (order by first_token_ms)
filter (where {fact}) as first_token_p99_ms
from model_requests mr where mr.started_at >= "
));
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_usage_filter(&mut query, filter, "mr");
let row = query
.build()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load request metrics"))?;
request_metrics_from_row(&row)
}
pub(crate) async fn request_metric_series(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<Vec<RequestMetricPoint>> {
filter.validate()?;
let granularity = granularity_for(range);
// 与 request_metrics 同一契约:结果计数覆盖全部请求,用量/延迟/成本
// 聚合仅统计用量事实。
let fact = completed_usage_fact_predicate("mr");
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
query.push_bind(granularity.sql_interval());
query.push(format!(
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
count(*)::bigint as request_count,
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
count(*) filter (where outcome = 'failed')::bigint as failure_count,
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
count(*) filter (where client_status_code between 400 and 499)::bigint
as caller_error_count,
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
as cache_write_tokens,
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
as reasoning_tokens,
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
as first_token_latency_sum,
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
count(latency_ms) filter (where {fact})::bigint as latency_count,
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
count(*) filter (where {fact} and input_tokens is not null)::bigint
as cache_eligible_request_count,
count(*) filter (where {fact} and input_tokens is not null
and cached_tokens > 0)::bigint
as cache_hit_request_count,
percentile_cont(0.50) within group (order by latency_ms)
filter (where {fact}) as latency_p50_ms,
percentile_cont(0.95) within group (order by latency_ms)
filter (where {fact}) as latency_p95_ms,
percentile_cont(0.99) within group (order by latency_ms)
filter (where {fact}) as latency_p99_ms,
percentile_cont(0.50) within group (order by first_token_ms)
filter (where {fact}) as first_token_p50_ms,
percentile_cont(0.95) within group (order by first_token_ms)
filter (where {fact}) as first_token_p95_ms,
percentile_cont(0.99) within group (order by first_token_ms)
filter (where {fact}) as first_token_p99_ms,
count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint
as provider_reported_count,
count(*) filter (where {fact} and cost_source = 'calculated')::bigint
as calculated_count,
count(*) filter (where {fact} and cost_source = 'unavailable')::bigint
as unavailable_count
from model_requests mr where mr.started_at >= "
));
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_usage_filter(&mut query, filter, "mr");
query.push(" group by bucket_start order by bucket_start");
let rows = query
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request metric series"))?;
let mut points = BTreeMap::new();
for row in rows {
let bucket_start = get(&row, "bucket_start")?;
points.insert(
bucket_start,
RequestMetricPoint {
bucket_start,
granularity,
metrics: request_metrics_from_row(&row)?,
cost_coverage: coverage_from_row(&row)?,
costs: Vec::new(),
},
);
}
let bucket_costs = request_costs_by_bucket(pool, range, filter, granularity).await?;
for (bucket, costs) in bucket_costs {
if let Some(point) = points.get_mut(&bucket) {
point.costs = costs;
}
}
fill_metric_gaps(range, granularity, points)
}
pub(crate) async fn calculated_usage_billing_facts(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<Vec<CalculatedUsageBillingFact>> {
filter.validate()?;
let granularity = granularity_for(range);
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
query.push_bind(granularity.sql_interval());
query.push(
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
mr.provider_kind, mr.upstream_model_id, mr.service_tier,
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
mr.cost_currency, mr.cost_amount::text as amount
from model_requests mr where mr.started_at >= ",
);
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
query.push(
" and mr.cost_source = 'calculated'
and mr.cost_amount is not null and mr.cost_currency is not null
and nullif(mr.provider_kind, '') is not null
and nullif(mr.upstream_model_id, '') is not null",
);
push_completed_usage_fact_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" order by bucket_start, mr.id");
query
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load calculated usage billing facts"))?
.iter()
.map(calculated_usage_billing_fact_from_row)
.collect()
}
pub(crate) async fn request_costs_by_bucket(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
granularity: ObservationGranularity,
) -> StoreResult<BTreeMap<DateTime<Utc>, Vec<CurrencyCostTotal>>> {
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
query.push_bind(granularity.sql_interval());
query.push(
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
mr.cost_currency, sum(mr.cost_amount)::text as amount
from model_requests mr
where mr.started_at >= ",
);
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
push_completed_usage_fact_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by bucket_start, mr.cost_currency order by bucket_start, mr.cost_currency");
let rows = query
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request costs by bucket"))?;
let mut result = BTreeMap::<DateTime<Utc>, Vec<CurrencyCostTotal>>::new();
for row in rows {
result
.entry(get(&row, "bucket_start")?)
.or_default()
.push(cost_from_row(&row)?);
}
Ok(result)
}
pub(crate) async fn attempt_metrics(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<AttemptMetrics> {
filter.validate()?;
let mut query = QueryBuilder::<Postgres>::new(
"with selected_requests as (
select mr.* from model_requests mr where mr.started_at >= ",
);
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_usage_filter(&mut query, filter, "mr");
query.push(
"), usage_facts as (
select * from selected_requests
where outcome = 'succeeded'
and downstream_committed_at is not null
and client_status_code between 200 and 399
), failures as (
select coalesce(oe.occurrence_count, 1)::bigint as occurrences,
oe.failure_kind, oe.status_code
from ops_events oe
join selected_requests sr on sr.id = oe.model_request_id
union all
select 1::bigint, coalesce(sr.error_kind, 'failed'),
coalesce(sr.upstream_status_code, sr.client_status_code)
from selected_requests sr
where sr.outcome = 'failed' and sr.attempt_count > 0
)
select coalesce((select sum(attempt_count) from selected_requests), 0)::bigint
as attempt_count,
coalesce((select count(*) from selected_requests
where outcome = 'succeeded' and attempt_count > 0), 0)::bigint
as success_count,
coalesce((select sum(occurrences) from failures), 0)::bigint as failure_count,
coalesce((select count(*) from selected_requests
where outcome = 'cancelled' and attempt_count > 0), 0)::bigint
as cancelled_count,
coalesce((select count(*) from selected_requests
where outcome = 'incomplete' and attempt_count > 0), 0)::bigint
as incomplete_count,
coalesce((select sum(occurrences) from failures
where failure_kind in ('rate_limited', 'quota_exhausted')
or status_code = 429), 0)::bigint as rate_limited_count,
coalesce((select sum(occurrences) from failures
where failure_kind in ('authentication', 'authorization',
'invalid_credential')
or status_code in (401, 403)), 0)::bigint as auth_failure_count,
coalesce((select sum(occurrences) from failures
where status_code between 500 and 599), 0)::bigint
as provider_5xx_count,
coalesce((select count(*) from usage_facts
where cost_source = 'provider_reported'), 0)::bigint
as provider_reported_count,
coalesce((select count(*) from usage_facts
where cost_source = 'calculated'), 0)::bigint
as calculated_count,
coalesce((select count(*) from usage_facts
where cost_source = 'unavailable'), 0)::bigint
as unavailable_count",
);
let row = query
.build()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load attempt metrics"))?;
Ok(AttemptMetrics {
attempt_count: unsigned(&row, "attempt_count")?,
success_count: unsigned(&row, "success_count")?,
failure_count: unsigned(&row, "failure_count")?,
cancelled_count: unsigned(&row, "cancelled_count")?,
incomplete_count: unsigned(&row, "incomplete_count")?,
rate_limited_count: unsigned(&row, "rate_limited_count")?,
auth_failure_count: unsigned(&row, "auth_failure_count")?,
provider_5xx_count: unsigned(&row, "provider_5xx_count")?,
cost_coverage: coverage_from_row(&row)?,
costs: request_costs(pool, range, filter).await?,
})
}
pub(crate) async fn request_costs(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<Vec<CurrencyCostTotal>> {
let mut query = QueryBuilder::<Postgres>::new(
"select mr.cost_currency, sum(mr.cost_amount)::text as amount
from model_requests mr where mr.started_at >= ",
);
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
push_completed_usage_fact_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by mr.cost_currency order by mr.cost_currency");
query
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request costs"))?
.iter()
.map(cost_from_row)
.collect()
}
pub(crate) async fn provider_account_metrics(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<ProviderAccountMetrics> {
let row = sqlx::query(
"with normalized as (
select enabled, availability, quota_limit_reached,
case
when not enabled then 'disabled'
when availability = 'banned' then 'banned'
when availability = 'quota_exhausted' then 'quota_exhausted'
when availability = 'invalid' then 'invalid'
when availability = 'expired'
or (access_token_expires_at is not null and access_token_expires_at <= $1)
then 'expired'
when quota_limit_reached then 'rate_limited'
when availability = 'unknown' then 'invalid'
else 'active'
end as status
from provider_accounts
)
select count(*)::bigint as total,
count(*) filter (where enabled)::bigint as enabled,
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
count(*) filter (where status = 'active')::bigint as active,
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
count(*) filter (where status = 'expired')::bigint as expired,
count(*) filter (where status = 'invalid')::bigint as invalid,
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
count(*) filter (where status = 'disabled')::bigint as disabled,
count(*) filter (where status = 'banned')::bigint as banned
from normalized",
)
.bind(observed_at)
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load provider account metrics"))?;
Ok(ProviderAccountMetrics {
total: unsigned(&row, "total")?,
enabled: unsigned(&row, "enabled")?,
unavailable: unsigned(&row, "unavailable")?,
active: unsigned(&row, "active")?,
rate_limited: unsigned(&row, "rate_limited")?,
expired: unsigned(&row, "expired")?,
invalid: unsigned(&row, "invalid")?,
quota_exhausted: unsigned(&row, "quota_exhausted")?,
disabled: unsigned(&row, "disabled")?,
banned: unsigned(&row, "banned")?,
})
}
pub(crate) async fn active_provider_account_ids(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<Vec<String>> {
sqlx::query(
"with normalized as (
select id,
case
when not enabled then 'disabled'
when availability = 'banned' then 'banned'
when availability = 'quota_exhausted' then 'quota_exhausted'
when availability in ('expired', 'invalid')
or (access_token_expires_at is not null and access_token_expires_at <= $1)
then 'expired'
else 'active'
end as status
from provider_accounts
)
select id
from normalized
where status = 'active'
order by id",
)
.bind(observed_at)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load active provider account IDs"))?
.iter()
.map(|row| get(row, "id"))
.collect()
}
pub(crate) async fn provider_observations(
pool: &PgPool,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<Vec<ProviderObservation>> {
let mut query = QueryBuilder::<Postgres>::new(
"select coalesce(mr.provider_kind, 'unrouted') as provider_kind,
count(*)::bigint as request_count,
coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count,
count(*) filter (where mr.outcome = 'failed')::bigint as failure_count,
coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens
from model_requests mr where mr.started_at >= ",
);
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_completed_usage_fact_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
let rows = query
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider observations"))?;
rows.iter()
.map(|row| {
Ok(ProviderObservation {
provider_kind: get(row, "provider_kind")?,
request_count: unsigned(row, "request_count")?,
attempt_count: unsigned(row, "attempt_count")?,
failure_count: unsigned(row, "failure_count")?,
total_tokens: unsigned(row, "total_tokens")?,
})
})
.collect()
}
pub(crate) fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<RequestMetrics> {
Ok(RequestMetrics {
request_count: unsigned(row, "request_count")?,
success_count: unsigned(row, "success_count")?,
failure_count: unsigned(row, "failure_count")?,
cancelled_count: unsigned(row, "cancelled_count")?,
incomplete_count: unsigned(row, "incomplete_count")?,
caller_error_count: unsigned(row, "caller_error_count")?,
input_tokens: unsigned(row, "input_tokens")?,
output_tokens: unsigned(row, "output_tokens")?,
cached_tokens: unsigned(row, "cached_tokens")?,
cache_write_tokens: unsigned(row, "cache_write_tokens")?,
reasoning_tokens: unsigned(row, "reasoning_tokens")?,
total_tokens: unsigned(row, "total_tokens")?,
first_token_latency_sum: unsigned(row, "first_token_latency_sum")?,
first_token_latency_count: unsigned(row, "first_token_latency_count")?,
latency_sum: unsigned(row, "latency_sum")?,
latency_count: unsigned(row, "latency_count")?,
max_latency_ms: optional_unsigned(row, "max_latency_ms")?,
min_latency_ms: optional_unsigned(row, "min_latency_ms")?,
cache_eligible_request_count: unsigned(row, "cache_eligible_request_count")?,
cache_hit_request_count: unsigned(row, "cache_hit_request_count")?,
latency_percentiles: LatencyPercentiles {
p50_ms: optional_percentile(row, "latency_p50_ms")?,
p95_ms: optional_percentile(row, "latency_p95_ms")?,
p99_ms: optional_percentile(row, "latency_p99_ms")?,
},
first_token_latency_percentiles: LatencyPercentiles {
p50_ms: optional_percentile(row, "first_token_p50_ms")?,
p95_ms: optional_percentile(row, "first_token_p95_ms")?,
p99_ms: optional_percentile(row, "first_token_p99_ms")?,
},
})
}
pub(crate) fn optional_percentile(
row: &sqlx::postgres::PgRow,
column: &str,
) -> StoreResult<Option<PercentileMilliseconds>> {
row.try_get::<Option<f64>, _>(column)
.map_err(|_| postgres_unavailable("decode latency percentile"))?
.map(PercentileMilliseconds::new)
.transpose()
}
pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CostCoverage> {
Ok(CostCoverage {
provider_reported_count: unsigned(row, "provider_reported_count")?,
calculated_count: unsigned(row, "calculated_count")?,
unavailable_count: unsigned(row, "unavailable_count")?,
})
}
pub(crate) fn granularity_for(range: ObservabilityRange) -> ObservationGranularity {
let seconds = range.end.signed_duration_since(range.start).num_seconds();
if seconds <= 2 * 24 * 60 * 60 {
ObservationGranularity::FifteenMinutes
} else if seconds <= 31 * 24 * 60 * 60 {
ObservationGranularity::Hour
} else {
ObservationGranularity::Day
}
}
pub(crate) fn fill_metric_gaps(
range: ObservabilityRange,
granularity: ObservationGranularity,
mut points: BTreeMap<DateTime<Utc>, RequestMetricPoint>,
) -> StoreResult<Vec<RequestMetricPoint>> {
let seconds = granularity.seconds();
let start_epoch = range.start.timestamp().div_euclid(seconds) * seconds;
let mut bucket = DateTime::from_timestamp(start_epoch, 0)
.ok_or_else(|| invalid("metric range start is outside supported timestamps"))?;
let step = TimeDelta::seconds(seconds);
let mut result = Vec::new();
while bucket < range.end {
result.push(points.remove(&bucket).unwrap_or(RequestMetricPoint {
bucket_start: bucket,
granularity,
metrics: RequestMetrics::default(),
cost_coverage: CostCoverage::default(),
costs: Vec::new(),
}));
bucket = bucket
.checked_add_signed(step)
.ok_or_else(|| invalid("metric range exceeds supported timestamps"))?;
}
Ok(result)
}
@@ -0,0 +1,11 @@
//! 观测查询族:dashboard / usage / ops / accounts。
mod accounts;
mod dashboard;
mod ops;
mod usage;
pub(crate) use accounts::*;
pub(crate) use dashboard::*;
pub(crate) use ops::*;
pub(crate) use usage::*;
@@ -0,0 +1,171 @@
//! Ops 错误查询族。
use super::super::*;
pub(crate) const OPS_ERRORS_CTE: &str = "with errors as (
select 'model_request'::text as source,
mr.id as event_id, mr.id as request_id,
nullif(mr.attempt_count, 0) as attempt_index,
mr.client_api_key_ref, 'model_request'::text as component, mr.operation,
mr.endpoint,
mr.provider_kind,
mr.provider_account_ref,
mr.provider_account_name_snapshot as provider_account_name,
mr.provider_account_email_snapshot as provider_account_email,
mr.provider_account_authentication_kind_snapshot
as provider_account_authentication_kind,
mr.upstream_model_id, mr.upstream_transport,
coalesce(mr.error_kind, 'failed') as failure_kind,
mr.client_status_code, mr.upstream_status_code,
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
mr.latency_ms, coalesce(mr.error_message, mr.error_kind, 'request failed') as message,
1::integer as occurrence_count,
coalesce(mr.completed_at, mr.started_at) as occurred_at,
'model_request:' || mr.id as stable_sort_id
from model_requests mr where mr.outcome = 'failed'
union all
select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index,
mr.client_api_key_ref, oe.component, oe.operation,
null::text as endpoint,
oe.provider_kind, oe.provider_account_ref,
oe.provider_account_name_snapshot as provider_account_name,
oe.provider_account_email_snapshot as provider_account_email,
oe.provider_account_authentication_kind_snapshot
as provider_account_authentication_kind,
oe.upstream_model_id, null::text, oe.failure_kind,
null::integer as client_status_code, oe.status_code as upstream_status_code,
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at,
'ops_event:' || oe.id
from ops_events oe left join model_requests mr on mr.id = oe.model_request_id
)";
pub(crate) async fn list_ops_errors(
pool: &PgPool,
query: OpsErrorQuery,
) -> StoreResult<OpsErrorPage> {
query.filter.validate()?;
let total = count_ops_errors(pool, query.range, &query.filter).await?;
let mut statement = QueryBuilder::<Postgres>::new(OPS_ERRORS_CTE);
statement.push(
" select source, event_id, request_id, attempt_index, client_api_key_ref,
component, operation, endpoint, provider_kind, provider_account_ref,
provider_account_name, provider_account_email,
provider_account_authentication_kind,
upstream_model_id, upstream_transport,
failure_kind, client_status_code, upstream_status_code,
provider_error_code, client_response_id,
upstream_request_id, latency_ms, message, occurrence_count, occurred_at,
stable_sort_id
from errors e where e.occurred_at >= ",
);
statement.push_bind(query.range.start);
statement.push(" and e.occurred_at < ");
statement.push_bind(query.range.end);
push_ops_filter(&mut statement, &query.filter);
if let Some(cursor) = &query.cursor {
statement.push(" and (e.occurred_at, e.stable_sort_id) < (");
statement.push_bind(cursor.observed_at);
statement.push(", ");
statement.push_bind(cursor.stable_id.clone());
statement.push(")");
}
statement.push(" order by e.occurred_at desc, e.stable_sort_id desc limit ");
statement.push_bind(i64::from(query.page_size.get()) + 1);
if query.cursor.is_none() {
let offset = u64::from(query.page.get() - 1)
.checked_mul(u64::from(query.page_size.get()))
.and_then(|value| i64::try_from(value).ok())
.ok_or_else(|| invalid("ops error page offset is too large"))?;
statement.push(" offset ");
statement.push_bind(offset);
}
let rows = statement
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("list ops errors"))?;
let mut items = rows
.iter()
.map(ops_error_from_row)
.collect::<StoreResult<Vec<_>>>()?;
let has_more = items.len() > usize::from(query.page_size.get());
if has_more {
items.pop();
}
let next_cursor = if has_more {
items
.last()
.map(|item| ObservabilityCursor::new(item.occurred_at, item.stable_sort_id.clone()))
.transpose()?
} else {
None
};
Ok(OpsErrorPage {
items,
total,
next_cursor,
})
}
pub(crate) async fn count_ops_errors(
pool: &PgPool,
range: ObservabilityRange,
filter: &OpsErrorFilter,
) -> StoreResult<u64> {
let mut statement = QueryBuilder::<Postgres>::new(OPS_ERRORS_CTE);
statement.push(" select count(*)::bigint from errors e where e.occurred_at >= ");
statement.push_bind(range.start);
statement.push(" and e.occurred_at < ");
statement.push_bind(range.end);
push_ops_filter(&mut statement, filter);
let total = statement
.build_query_scalar::<i64>()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("count ops errors"))?;
to_u64(total)
}
pub(crate) fn push_ops_filter(statement: &mut QueryBuilder<Postgres>, filter: &OpsErrorFilter) {
for (column, value) in [
("client_api_key_ref", &filter.client_api_key_ref),
("request_id", &filter.request_id),
("provider_account_ref", &filter.provider_account_ref),
("provider_kind", &filter.provider_kind),
("operation", &filter.operation),
("upstream_transport", &filter.transport),
("upstream_request_id", &filter.upstream_request_id),
("failure_kind", &filter.failure_kind),
] {
if let Some(value) = value {
statement.push(format!(" and e.{column} = "));
statement.push_bind(value.clone());
}
}
if let Some(value) = &filter.response_id {
statement.push(" and e.client_response_id = ");
statement.push_bind(value.as_bytes().to_vec());
}
if let Some(model) = &filter.model {
statement.push(" and e.upstream_model_id = ");
statement.push_bind(model.clone());
}
if let Some(index) = filter.attempt_index {
statement.push(" and e.attempt_index = ");
statement.push_bind(i32::try_from(index).unwrap_or(i32::MAX));
}
if let Some(status) = filter.status_code {
statement.push(" and e.upstream_status_code = ");
statement.push_bind(i32::from(status));
}
if let Some(search) = &filter.search {
statement.push(
" and lower(concat_ws(' ', e.event_id, e.request_id, e.client_api_key_ref,
e.component, e.operation, e.provider_kind, e.provider_account_ref,
e.upstream_model_id, e.failure_kind, e.provider_error_code,
e.upstream_request_id, e.message)) like ",
);
statement.push_bind(format!("%{}%", search.to_lowercase()));
}
}

Some files were not shown because too many files have changed in this diff Show More