Compare commits

...
62 Commits
Author SHA1 Message Date
zyycnandCodex f30652e113 fix: scan release images for target platform
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 13:34:51 +08:00
zyycnandCodex 021e87c59e chore: bump version to 2.0.0
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 13:18:09 +08:00
zyycnandCodex f867f13ce7 fix: upgrade GitHub Actions runtimes
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 12:59:49 +08:00
zyycnandCodex d7bb5e4ab2 fix: configure redis memory overcommit in container CI
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 12:52:01 +08:00
zyycnandCodex 5d0df748c5 docs: align repository skills with current architecture
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 12:45:15 +08:00
zyycnandCodex b3b5377683 refactor: remove dead code and refresh documentation
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 12:33:59 +08:00
zyycnandCodex 203fc1b03e fix: harden persistence and runtime boundaries
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 10:07:21 +08:00
zyycnandCodex b8e5b6eade fix: harden response recovery and account failover
Co-authored-by: Codex <noreply@openai.com>
2026-07-12 01:27:28 +08:00
zyycnandCodex 62df4d714c refactor: tighten backend dependency graph
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 18:06:16 +08:00
zyycnandCodex 4dcd9dc31d feat: harden deployment and CI pipeline
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 17:18:17 +08:00
zyycnandCodex 49120d0efb docs: define hardened deployment and CI design
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 16:00:04 +08:00
zyycnandCodex 4d48a7bca1 fix: make concurrent lifecycles cancellation-safe
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 15:20:19 +08:00
zyycnandCodex 3c33e93da9 fix: unify pagination and error contracts
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 13:58:41 +08:00
zyycnandCodex 1652b60769 feat: harden logging and retention boundaries
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 04:07:34 +08:00
zyycnandCodex 899454e6b5 docs: define bounded logging and retention design
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 03:07:05 +08:00
zyycnandCodex 7a6c6927bf refactor: complete architecture and naming alignment
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 02:31:26 +08:00
zyycnandCodex 7cb186c4ae refactor: conform backend to authoritative architecture
Co-authored-by: Codex <noreply@openai.com>
2026-07-11 00:43:47 +08:00
zyycnandCodex fe5494a614 refactor: align implementation with authoritative architecture
Co-authored-by: Codex <noreply@openai.com>
2026-07-10 22:53:27 +08:00
ZyyandCodex 61cdaf753e fix: harden responses websocket recovery
Co-authored-by: Codex <noreply@openai.com>
2026-07-10 20:30:40 +08:00
ZyyandCodex f06530278d fix: align postgresql redis migration contracts
Co-authored-by: Codex <noreply@openai.com>
2026-07-10 19:35:58 +08:00
ZyyandCodex 45e4b36225 refactor: complete postgresql redis migration
Co-authored-by: Codex <noreply@openai.com>
2026-07-10 19:00:14 +08:00
ZyyandCodex 68e76d8138 fix: constrain account token refresh lifecycle
Co-authored-by: Codex <noreply@openai.com>
2026-07-10 12:58:04 +08:00
zyycnandCodex 89ff4be281 docs: document postgresql redis storage target
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 22:39:22 +08:00
ZyyandCodex 788e587259 fix: preserve account claims during token refresh
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 17:48:50 +08:00
ZyyandCodex cbf9ce91a1 chore: bump version to 1.0.11
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 15:44:19 +08:00
ZyyandCodex 71e51a868f fix: keep quota exhausted status during token refresh
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 15:41:42 +08:00
ZyyandCodex 3c9ec50bf2 chore: bump version to 1.0.10
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 13:08:08 +08:00
ZyyandCodex c1b81809a6 refactor: derive account refresh status from leases
Co-authored-by: Codex <noreply@openai.com>
2026-07-09 13:06:54 +08:00
Zyy 939f392438 docs: refresh architecture and database docs 2026-07-08 21:35:26 +08:00
ZyyandCodex ff0de4742d chore: bump version to 1.0.9
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 18:06:08 +08:00
ZyyandCodex 44f45101b8 test: assert account quota local usage
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 17:44:01 +08:00
ZyyandCodex a240668a47 fix: polish sidebar focus state
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 16:45:52 +08:00
ZyyandCodex 30f354965c fix: align account quota usage tracking
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 16:37:32 +08:00
ZyyandCodex 3b462f7ecf refactor: remove trusted proxy config
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 15:22:23 +08:00
ZyyandCodex 5682d4fbe9 test: simplify ops error log snapshot helper
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 14:55:05 +08:00
ZyyandCodex 146776c742 fix: separate ops errors from usage records
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 14:04:49 +08:00
ZyyandCodex e8c14d4324 fix: complete update log streaming
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 13:58:28 +08:00
zyycnandCodex 5483e37b00 refactor: extract upstream scheduler
Co-authored-by: Codex <noreply@openai.com>
2026-07-08 13:56:27 +08:00
ZyyandCodex daad7c746c fix: strengthen response account affinity
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 18:31:50 +08:00
ZyyandCodex 6e24037c4d chore: bump version to 1.0.8
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 17:06:08 +08:00
ZyyandCodex 95daf7093e feat: add smart account scheduling
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 17:00:56 +08:00
ZyyandCodex 2e121c0670 fix: hot reload fingerprint updates
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 15:01:28 +08:00
ZyyandCodex 66916ab8b2 chore: bump version to 1.0.7
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 14:04:19 +08:00
ZyyandCodex 2945523140 fix: clean account routing and affinity fallback
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 14:01:35 +08:00
ZyyandCodex f375d2d77c fix: allow large OpenAI request bodies
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 14:00:04 +08:00
zyycnandCodex c6e1b347be chore: bump version to 1.0.6
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:54:27 +08:00
zyycnandCodex 1ac9803c34 fix: stop recording account state usage events
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:54:14 +08:00
zyycnandCodex 352b781340 chore: add engineering standards skill
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:10:02 +08:00
zyycnandCodex 62a6dfc489 chore: bump version to 1.0.5
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 00:16:26 +08:00
zyycnandCodex e91f91b542 test: move source tests to integration suite
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 00:15:30 +08:00
zyycnandCodex 380556369b fix: harden runtime diagnostics and quota refresh
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 23:59:27 +08:00
ZyyandCodex 595cda541d style: apply cargo fmt
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 17:55:54 +08:00
ZyyandCodex 90ed8bf6bc chore: bump version to 1.0.4
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 17:49:48 +08:00
ZyyandCodex 1101eee628 fix: align quota window accounting
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 17:49:21 +08:00
ZyyandCodex 0e952c9d83 fix: enforce websocket first token retries
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 17:49:15 +08:00
ZyyandCodex 875d610812 fix: refine dashboard metrics
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 16:53:31 +08:00
ZyyandCodex 9c37d262d1 fix: retry websocket first token stalls
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 16:53:07 +08:00
ZyyandCodex bd95adadf9 fix: align dashboard daily metrics
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 13:19:02 +08:00
zyycnandCodex 6e6627b0c9 docs: refresh project documentation
Co-authored-by: Codex <noreply@openai.com>
2026-07-05 22:39:26 +08:00
zyycnandCodex 3579c28b75 chore: bump version to 1.0.3
Co-authored-by: Codex <noreply@openai.com>
2026-07-05 22:27:00 +08:00
zyycnandCodex 2143de2791 feat: animate dashboard metrics
Co-authored-by: Codex <noreply@openai.com>
2026-07-05 22:26:30 +08:00
zyycnandCodex 298357f97c feat: refine admin console ergonomics
Co-authored-by: Codex <noreply@openai.com>
2026-07-05 22:15:52 +08:00
493 changed files with 44567 additions and 34432 deletions
+54
View File
@@ -0,0 +1,54 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
day: monday
time: '09:00'
timezone: Asia/Shanghai
groups:
minor-and-patch:
update-types:
- minor
- patch
- package-ecosystem: cargo
directory: /backend
schedule:
interval: weekly
day: monday
time: '09:10'
timezone: Asia/Shanghai
groups:
minor-and-patch:
update-types:
- minor
- patch
- package-ecosystem: npm
directory: /frontend
schedule:
interval: weekly
day: monday
time: '09:20'
timezone: Asia/Shanghai
groups:
minor-and-patch:
update-types:
- minor
- patch
- package-ecosystem: docker
directory: /deploy
schedule:
interval: weekly
day: monday
time: '09:30'
timezone: Asia/Shanghai
groups:
minor-and-patch:
update-types:
- minor
- patch
+244
View File
@@ -0,0 +1,244 @@
name: Reusable Container
on:
workflow_call:
inputs:
ref:
description: Exact commit SHA to build and validate
required: true
type: string
permissions:
contents: read
security-events: write
jobs:
container:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
COMPOSE_PROJECT_NAME: cpr-ci-${{ github.run_id }}-${{ github.run_attempt }}
CPR_IMAGE: codex-proxy-rs:ci-${{ inputs.ref }}
CPR_ADMIN_DEFAULT_PASSWORD: 'admin @:/?#%2F ci password'
CPR_POSTGRES_PASSWORD: 'pg @:/?#%2F ci'
CPR_REDIS_PASSWORD: 'redis @:/?#%2F ci'
CPR_SMOKE_PORT: '18080'
TRIVY_QUIET: 'true'
TRIVY_SKIP_VERSION_CHECK: 'true'
steps:
- name: Define isolated runtime paths
shell: bash
run: |
{
echo "CPR_CONFIG_FILE=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/config.yaml"
echo "CPR_DATA_DIR=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/data"
echo "CPR_LOG_DIR=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/logs"
echo "DIAGNOSTICS_DIR=$RUNNER_TEMP/codex-proxy-rs-diagnostics"
} >> "$GITHUB_ENV"
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.ref }}
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Build runtime image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: deploy/Dockerfile
target: runtime
load: true
tags: ${{ env.CPR_IMAGE }}
cache-from: type=gha,scope=runtime-ci
cache-to: type=gha,mode=max,scope=runtime-ci
- name: Generate Trivy SARIF
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: ${{ env.CPR_IMAGE }}
version: v0.72.0
format: sarif
output: trivy-results.sarif
severity: HIGH,CRITICAL
vuln-type: os,library
scanners: vuln
ignore-unfixed: true
exit-code: '0'
- name: Upload Trivy SARIF
if: >-
${{ hashFiles('trivy-results.sarif') != '' &&
(github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository) }}
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
with:
sarif_file: trivy-results.sarif
category: runtime-container
- name: Enforce Trivy vulnerability gate
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: ${{ env.CPR_IMAGE }}
version: v0.72.0
format: table
severity: HIGH,CRITICAL
vuln-type: os,library
scanners: vuln
ignore-unfixed: true
exit-code: '1'
- name: Prepare isolated runtime
shell: bash
run: |
set -euo pipefail
runtime_dir="$(dirname "$CPR_CONFIG_FILE")"
mkdir -p "$runtime_dir" "$DIAGNOSTICS_DIR"
sudo install -d -m 0770 -o "$(id -u)" -g 10001 "$CPR_DATA_DIR" "$CPR_LOG_DIR"
cp deploy/config.example.yaml "$CPR_CONFIG_FILE"
sudo chown "$(id -u):10001" "$CPR_CONFIG_FILE"
chmod 0640 "$CPR_CONFIG_FILE"
- name: Validate Docker Compose
shell: bash
run: |
set -euo pipefail
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml config --quiet
- name: Configure Redis host memory overcommit
shell: bash
run: |
set -euo pipefail
sudo sysctl -w vm.overcommit_memory=1
test "$(sysctl -n vm.overcommit_memory)" = 1
- name: Verify PostgreSQL and Redis volume persistence
shell: bash
run: |
set -euo pipefail
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
"${compose[@]}" up -d --no-build --wait postgres redis
"${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
"CREATE TABLE cpr_ci_persistence_probe (value text PRIMARY KEY); INSERT INTO cpr_ci_persistence_probe VALUES ('persisted');"
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
redis-cli SET cpr:ci:persistence-probe persisted
"${compose[@]}" up -d --no-build --wait --force-recreate postgres redis
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
psql -At -U codex_proxy -d codex_proxy -c \
"SELECT value FROM cpr_ci_persistence_probe;")" = persisted
test "$("${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
redis-cli --raw GET cpr:ci:persistence-probe | tr -d '\r')" = persisted
"${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
"DROP TABLE cpr_ci_persistence_probe;"
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
redis-cli DEL cpr:ci:persistence-probe
- name: Start and verify application
shell: bash
run: |
set -euo pipefail
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
"${compose[@]}" up -d --no-build --wait codex-proxy-rs
status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
"http://127.0.0.1:${CPR_SMOKE_PORT}/healthz")"
test "$status" = 204
test "$("${compose[@]}" exec -T codex-proxy-rs id -u | tr -d '\r')" = 10001
test "$("${compose[@]}" exec -T codex-proxy-rs id -g | tr -d '\r')" = 10001
"${compose[@]}" exec -T codex-proxy-rs sh -c \
'touch /app/data/.ci-write /app/logs/.ci-write && rm /app/data/.ci-write /app/logs/.ci-write'
- name: Verify clean logs and graceful shutdown
shell: bash
run: |
set -euo pipefail
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
"${compose[@]}" stop --timeout 30 codex-proxy-rs
container_id="$("${compose[@]}" ps -a -q codex-proxy-rs)"
test -n "$container_id"
test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = 0
"${compose[@]}" logs --no-color --timestamps codex-proxy-rs \
> "$DIAGNOSTICS_DIR/application.log"
"${compose[@]}" logs --no-color --timestamps postgres \
> "$DIAGNOSTICS_DIR/postgres.log"
"${compose[@]}" logs --no-color --timestamps redis \
> "$DIAGNOSTICS_DIR/redis.log"
if grep --extended-regexp --ignore-case \
'"level"[[:space:]]*:[[:space:]]*"(warn|error|fatal|panic)"|(^|[^[:alnum:]_])panic([^[:alnum:]_]|$)' \
"$DIAGNOSTICS_DIR/application.log" \
|| grep --extended-regexp --ignore-case \
'(^|[[:space:]])(warning|error|fatal|panic):' \
"$DIAGNOSTICS_DIR/postgres.log" \
|| grep --extended-regexp --ignore-case \
'[0-9]+:[A-Z][[:space:]].*[[:space:]]#[[:space:]]|(^|[|[:space:]])(warning|error|fatal|panic)(:|[[:space:]])' \
"$DIAGNOSTICS_DIR/redis.log"; then
echo "container logs contain a warning or error severity" >&2
exit 1
fi
- name: Collect container diagnostics
if: always()
shell: bash
run: |
set +e
mkdir -p "$DIAGNOSTICS_DIR"
raw_dir="$(mktemp -d)"
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
"${compose[@]}" config > "$raw_dir/compose-config.yaml" 2>&1
"${compose[@]}" ps -a > "$raw_dir/compose-ps.txt" 2>&1
"${compose[@]}" logs --no-color --timestamps > "$raw_dir/compose.log" 2>&1
mapfile -t container_ids < <("${compose[@]}" ps -a -q)
if [ "${#container_ids[@]}" -gt 0 ]; then
docker inspect "${container_ids[@]}" > "$raw_dir/docker-inspect.json" 2>&1
fi
RAW_DIR="$raw_dir" OUTPUT_DIR="$DIAGNOSTICS_DIR" ruby <<'RUBY'
raw_dir = ENV.fetch("RAW_DIR")
output_dir = ENV.fetch("OUTPUT_DIR")
secrets = [
ENV.fetch("CPR_ADMIN_DEFAULT_PASSWORD"),
ENV.fetch("CPR_POSTGRES_PASSWORD"),
ENV.fetch("CPR_REDIS_PASSWORD"),
]
encoded_secrets = secrets.map do |secret|
secret.encode("UTF-8").bytes.map do |byte|
character = byte.chr
character.match?(/[A-Za-z0-9]/) ? character : format("%%%02X", byte)
end.join
end
Dir.children(raw_dir).each do |name|
content = File.binread(File.join(raw_dir, name))
(secrets + encoded_secrets).each do |secret|
content = content.gsub(secret, "<redacted>")
end
File.binwrite(File.join(output_dir, name), content)
end
RUBY
rm -rf "$raw_dir"
cp trivy-results.sarif "$DIAGNOSTICS_DIR/" 2>/dev/null || true
- name: Upload container diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
if-no-files-found: warn
retention-days: 7
- name: Remove isolated runtime
if: always()
shell: bash
run: |
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml \
down --volumes --remove-orphans || true
sudo rm -rf "$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+137
View File
@@ -0,0 +1,137 @@
name: Reusable Quality
on:
workflow_call:
inputs:
ref:
description: Exact commit SHA to validate
required: true
type: string
upload_frontend:
description: Upload the built frontend for release jobs
required: false
default: false
type: boolean
permissions:
contents: read
jobs:
backend:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
CARGO_TERM_COLOR: always
CPR_TEST_DATABASE_URL: postgres://codex_proxy:codex_proxy@127.0.0.1:5432/codex_proxy
CPR_TEST_REDIS_URL: redis://127.0.0.1:6379
RUSTFLAGS: -D warnings
services:
postgres:
image: postgres:18-bookworm@sha256:d9c83446333daec3f0588cc709adb80c26090b7f9f0f7ec8d43c243385d79818
env:
POSTGRES_DB: codex_proxy
POSTGRES_USER: codex_proxy
POSTGRES_PASSWORD: codex_proxy
POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U codex_proxy -d codex_proxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
redis:
image: redis:8-alpine@sha256:9d317178eceac8454a2284a9e6df2466b93c745529947f0cd42a0fa9609d7005
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 10
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.ref }}
- name: Setup Rust
uses: dtolnay/rust-toolchain@eac0f66a48bc4b70a10b9acd4c4e930f835d95ff # 1.95
with:
components: rustfmt,clippy
- name: Cache Rust
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
with:
workspaces: backend -> target
- name: Rust format
working-directory: backend
run: cargo fmt --check
- name: Rust clippy
working-directory: backend
run: cargo clippy --all-targets --all-features --locked -- -D warnings
- name: Rust tests
working-directory: backend
run: cargo test --test main --locked
frontend:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
CI: 'true'
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.ref }}
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
version: 11.7.0
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
cache-dependency-path: frontend/pnpm-lock.yaml
- name: Install frontend dependencies
run: pnpm --dir frontend install --frozen-lockfile
- name: Frontend format
run: pnpm --dir frontend format:check
- name: Frontend build
run: pnpm --dir frontend build
- name: Upload frontend artifact
if: inputs.upload_frontend
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: frontend-dist
path: frontend/dist
if-no-files-found: error
retention-days: 1
workflow-lint:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.ref }}
- name: Lint GitHub Actions workflows
run: >-
docker run --rm
--volume "$PWD:/repo"
--workdir /repo
rhysd/actionlint:1.7.7@sha256:887a259a5a534f3c4f36cb02dca341673c6089431057242cdc931e9f133147e9
-color
.github/workflows/*.yml
+33 -59
View File
@@ -10,73 +10,47 @@ permissions:
contents: read
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
group: ci-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
jobs:
backend:
changes:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
container: ${{ steps.filter.outputs.container }}
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.95
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
components: rustfmt,clippy
ref: ${{ github.event.pull_request.head.sha || github.sha }}
fetch-depth: 0
- name: Cache Rust
uses: Swatinem/rust-cache@v2.9.1
- name: Detect container changes
id: filter
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
with:
workspaces: backend -> target
filters: |
container:
- 'backend/**'
- 'frontend/**'
- 'deploy/**'
- '.dockerignore'
- 'release/version.yaml'
- '.github/workflows/_container.yml'
- '.github/workflows/ci.yml'
- name: Rust format
working-directory: backend
run: cargo fmt --check
quality:
uses: ./.github/workflows/_quality.yml
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Rust clippy
working-directory: backend
run: cargo clippy --all-targets --all-features --locked -- -D warnings
- name: Rust tests
working-directory: backend
run: cargo test --test main --locked
frontend:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 11.7.0
- name: Setup Node
uses: actions/setup-node@v6
with:
node-version: 24
cache: pnpm
cache-dependency-path: frontend/pnpm-lock.yaml
- name: Install frontend dependencies
run: pnpm --dir frontend install --frozen-lockfile
- name: Frontend format
run: pnpm --dir frontend format:check
- name: Frontend build
run: pnpm --dir frontend build
deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Validate Docker Compose
run: docker compose -f deploy/docker-compose.yml config
- name: Build runtime image
run: docker build -f deploy/Dockerfile --target runtime -t codex-proxy-rs:ci .
container:
needs: [changes, quality]
if: needs.changes.outputs.container == 'true'
permissions:
contents: read
security-events: write
uses: ./.github/workflows/_container.yml
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
+93 -92
View File
@@ -25,6 +25,7 @@ env:
jobs:
release-info:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
tag: ${{ steps.version.outputs.tag }}
version: ${{ steps.version.outputs.version }}
@@ -35,10 +36,11 @@ jobs:
non_docker_matrix: ${{ steps.version.outputs.non_docker_matrix }}
docker_platforms: ${{ steps.version.outputs.docker_platforms }}
previous_tag: ${{ steps.version.outputs.previous_tag }}
prerelease: ${{ steps.version.outputs.prerelease }}
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
fetch-depth: 0
@@ -151,62 +153,33 @@ jobs:
puts "non_docker_matrix=#{non_docker_matrix}"
puts "docker_platforms=#{docker_platforms.join(",")}"
puts "previous_tag=#{previous_tag}"
puts "prerelease=#{version.split("+", 2).first.include?("-")}"
RUBY
build-frontend:
quality:
needs: [release-info]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
with:
ref: ${{ needs.release-info.outputs.tag }}
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 11.7.0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24
cache: pnpm
cache-dependency-path: frontend/pnpm-lock.yaml
- name: Install dependencies
run: pnpm install --frozen-lockfile
working-directory: frontend
- name: Build frontend
run: pnpm build
working-directory: frontend
- name: Upload frontend artifact
uses: actions/upload-artifact@v7
with:
name: frontend-dist
path: frontend/dist
if-no-files-found: error
retention-days: 1
uses: ./.github/workflows/_quality.yml
with:
ref: ${{ needs.release-info.outputs.git_sha }}
upload_frontend: true
build-docker-binary:
name: build-docker-binary (${{ matrix.os }}-${{ matrix.arch }})
needs: [release-info]
needs: [release-info, quality]
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.release-info.outputs.docker_matrix) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.release-info.outputs.tag }}
ref: ${{ needs.release-info.outputs.git_sha }}
- name: Cache Cargo
uses: actions/cache@v6
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/.cargo/registry
@@ -232,9 +205,8 @@ jobs:
-v "$HOME/.cargo:/cargo" \
-v "$PWD/..:/workspace" \
-w /workspace/backend \
rust:1.95-bookworm \
rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 \
cargo build --release --locked --target "${{ matrix.target }}" --bin codex-proxy-rs
mkdir -p "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}"
cp "target/${{ matrix.target }}/release/${{ matrix.binary }}" "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/${{ matrix.binary }}"
working-directory: backend
env:
@@ -244,7 +216,7 @@ jobs:
CPR_BUILD_TYPE: release
- name: Upload binary artifact
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
@@ -253,20 +225,26 @@ jobs:
build-binary:
name: build-binary (${{ matrix.os }}-${{ matrix.arch }})
needs: [release-info]
needs: [release-info, quality]
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.release-info.outputs.non_docker_matrix) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.release-info.outputs.tag }}
ref: ${{ needs.release-info.outputs.git_sha }}
- name: Setup Rust
uses: dtolnay/rust-toolchain@eac0f66a48bc4b70a10b9acd4c4e930f835d95ff # 1.95
with:
targets: ${{ matrix.target }}
- name: Cache Cargo
uses: actions/cache@v6
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/.cargo/registry
@@ -276,10 +254,6 @@ jobs:
restore-keys: |
cargo-${{ runner.os }}-${{ matrix.target }}-
- name: Install Rust target
run: rustup target add ${{ matrix.target }}
working-directory: backend
- name: Build binary
shell: bash
run: |
@@ -294,7 +268,7 @@ jobs:
CPR_BUILD_TYPE: release
- name: Upload binary artifact
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
@@ -302,8 +276,9 @@ jobs:
retention-days: 1
package-assets:
needs: [release-info, build-frontend, build-docker-binary, build-binary]
needs: [release-info, quality, build-docker-binary, build-binary]
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
id-token: write
@@ -311,18 +286,18 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.release-info.outputs.tag }}
ref: ${{ needs.release-info.outputs.git_sha }}
- name: Download frontend artifact
uses: actions/download-artifact@v8
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: frontend-dist
path: dist/frontend
- name: Download binary artifacts
uses: actions/download-artifact@v8
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: codex-proxy-rs-*
path: dist/binaries
@@ -407,12 +382,12 @@ jobs:
PY
- name: Generate release asset provenance
uses: actions/attest@v4
uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1
with:
subject-path: dist/release/*
- name: Upload release assets
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-assets
path: dist/release
@@ -421,30 +396,34 @@ jobs:
build-docker-image:
name: build-docker-image (${{ matrix.dockerPlatform }})
needs: [release-info, build-frontend, build-docker-binary]
needs: [release-info, quality, build-docker-binary]
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.release-info.outputs.docker_matrix) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
permissions:
contents: read
packages: write
id-token: write
env:
TRIVY_QUIET: 'true'
TRIVY_SKIP_VERSION_CHECK: 'true'
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.release-info.outputs.tag }}
ref: ${{ needs.release-info.outputs.git_sha }}
- name: Download frontend artifact
uses: actions/download-artifact@v8
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: frontend-dist
path: dist/docker/web/dist
- name: Download Docker binary artifacts
uses: actions/download-artifact@v8
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
@@ -476,10 +455,10 @@ jobs:
PY
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to GHCR
uses: docker/login-action@v4
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -487,7 +466,7 @@ jobs:
- name: Build and push app image
id: build-and-push
uses: docker/build-push-action@v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: deploy/Dockerfile
@@ -505,13 +484,26 @@ jobs:
org.opencontainers.image.version=${{ needs.release-info.outputs.version }}
org.opencontainers.image.revision=${{ needs.release-info.outputs.git_sha }}
org.opencontainers.image.source=https://github.com/zyycn/codex-proxy-rs
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}-${{ matrix.arch }}
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}-${{ matrix.arch }}
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}-${{ matrix.arch }}
- name: Scan exact app image digest
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
env:
TRIVY_PLATFORM: ${{ matrix.dockerPlatform }}
with:
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }}
version: v0.72.0
format: table
severity: HIGH,CRITICAL
vuln-type: os,library
scanners: vuln
ignore-unfixed: true
exit-code: '1'
publish:
needs: [release-info, package-assets, build-docker-image]
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: write
packages: write
@@ -520,24 +512,24 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.release-info.outputs.tag }}
ref: ${{ needs.release-info.outputs.git_sha }}
- name: Download release assets
uses: actions/download-artifact@v8
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-assets
path: dist/release
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Install Cosign
uses: sigstore/cosign-installer@v4.1.2
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Login to GHCR
uses: docker/login-action@v4
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -550,6 +542,7 @@ jobs:
VERSION: ${{ needs.release-info.outputs.version }}
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
DOCKER_PLATFORMS: ${{ needs.release-info.outputs.docker_platforms }}
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
run: |
set -euo pipefail
@@ -560,7 +553,7 @@ jobs:
platform="${platform//[[:space:]]/}"
[ -n "${platform}" ] || continue
arch="${platform##*/}"
sources+=("${image}:${VERSION}-${arch}")
sources+=("${image}:sha-${GIT_SHA}-${arch}")
done
if [ "${#sources[@]}" -eq 0 ]; then
@@ -568,11 +561,14 @@ jobs:
exit 1
fi
docker buildx imagetools create \
-t "${image}:${VERSION}" \
-t "${image}:latest" \
-t "${image}:sha-${GIT_SHA}" \
"${sources[@]}"
target_args=(
-t "${image}:${VERSION}"
-t "${image}:sha-${GIT_SHA}"
)
if [ "$PRERELEASE" != true ]; then
target_args+=(-t "${image}:latest")
fi
docker buildx imagetools create "${target_args[@]}" "${sources[@]}"
inspect_output="$(docker buildx imagetools inspect "${image}:${VERSION}")"
platform_list="$(awk '$1 == "Platform:" { print $2 }' <<< "${inspect_output}")"
@@ -592,7 +588,7 @@ jobs:
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
- name: Generate container provenance attestation
uses: actions/attest@v4
uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
subject-digest: ${{ steps.manifest.outputs.digest }}
@@ -602,10 +598,9 @@ jobs:
shell: bash
env:
DIGEST: ${{ steps.manifest.outputs.digest }}
IMAGE_TAGS: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}
VERSION: ${{ needs.release-info.outputs.version }}
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
run: |
set -euo pipefail
if [ -z "${DIGEST}" ]; then
@@ -613,20 +608,26 @@ jobs:
exit 1
fi
image="${REGISTRY}/${IMAGE_NAME}"
tags=("${image}:${VERSION}" "${image}:sha-${GIT_SHA}")
if [ "$PRERELEASE" != true ]; then
tags+=("${image}:latest")
fi
images=()
while IFS= read -r tag; do
[ -n "${tag}" ] || continue
for tag in "${tags[@]}"; do
images+=("${tag}@${DIGEST}")
done <<< "${IMAGE_TAGS}"
done
cosign sign --yes "${images[@]}"
- name: Publish GitHub Release
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
with:
tag_name: ${{ needs.release-info.outputs.tag }}
generate_release_notes: true
previous_tag: ${{ needs.release-info.outputs.previous_tag }}
prerelease: ${{ needs.release-info.outputs.prerelease == 'true' }}
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
body: |
## 📥 Installation
+20 -26
View File
@@ -12,7 +12,7 @@ permissions:
contents: read
concurrency:
group: security-scan-${{ github.workflow }}-${{ github.ref }}
group: security-scan-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
jobs:
@@ -21,18 +21,20 @@ jobs:
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.95
uses: dtolnay/rust-toolchain@eac0f66a48bc4b70a10b9acd4c4e930f835d95ff # 1.95
- name: Cache Rust
uses: Swatinem/rust-cache@v2.9.1
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
with:
workspaces: backend -> target
- name: Install cargo-audit
run: cargo install cargo-audit --locked
run: cargo install cargo-audit --version 0.22.2 --locked
- name: Run cargo audit
working-directory: backend
@@ -43,15 +45,17 @@ jobs:
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Setup pnpm
uses: pnpm/action-setup@v6
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
version: 11.7.0
- name: Setup Node
uses: actions/setup-node@v6
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
@@ -63,21 +67,11 @@ jobs:
- name: Run pnpm audit
run: pnpm --dir frontend audit --prod --audit-level=high
docker-security:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Build runtime image
run: docker build -f deploy/Dockerfile --target runtime -t codex-proxy-rs:security-scan .
- name: Scan runtime image
uses: aquasecurity/trivy-action@v0.36.0
with:
image-ref: codex-proxy-rs:security-scan
severity: HIGH,CRITICAL
vuln-type: os,library
ignore-unfixed: true
exit-code: '1'
container-security:
if: github.event_name == 'schedule'
permissions:
contents: read
security-events: write
uses: ./.github/workflows/_container.yml
with:
ref: ${{ github.sha }}
+1
View File
@@ -10,6 +10,7 @@
# Frontend dependencies and build output.
/frontend/node_modules/
/frontend/dist/
.pnpm-store
# Application runtime state.
*.runtime
+173 -151
View File
@@ -1,204 +1,226 @@
<h1 align="center">Codex Proxy RS</h1>
<div align="center">
<h1>Codex Proxy RS</h1>
<p>OpenAI Responses 兼容的 Codex 多账号网关</p>
<p>
<a href="https://www.rust-lang.org/"><img alt="Rust 1.95" src="https://img.shields.io/badge/Rust-1.95-000?style=flat-square&amp;logo=rust" /></a>
<a href="https://github.com/tokio-rs/axum"><img alt="Axum 0.8" src="https://img.shields.io/badge/Axum-0.8-2f7d95?style=flat-square" /></a>
<a href="https://vuejs.org/"><img alt="Vue 3.5" src="https://img.shields.io/badge/Vue-3.5-42b883?style=flat-square&amp;logo=vuedotjs&amp;logoColor=white" /></a>
<a href="https://vite.dev/"><img alt="Vite 8" src="https://img.shields.io/badge/Vite-8-646cff?style=flat-square&amp;logo=vite&amp;logoColor=white" /></a>
<a href="https://www.postgresql.org/"><img alt="PostgreSQL 18" src="https://img.shields.io/badge/PostgreSQL-18-4169e1?style=flat-square&amp;logo=postgresql&amp;logoColor=white" /></a>
<a href="https://redis.io/"><img alt="Redis 8" src="https://img.shields.io/badge/Redis-8-ff4438?style=flat-square&amp;logo=redis&amp;logoColor=white" /></a>
<a href="https://opensource.org/license/mit"><img alt="MIT License" src="https://img.shields.io/badge/License-MIT-blue?style=flat-square" /></a>
</p>
<p>
<a href="#快速部署">快速部署</a>
·
<a href="#客户端接入">客户端接入</a>
·
<a href="docs/architecture.md">架构说明</a>
</p>
</div>
<p align="center">
OpenAI 兼容的 ChatGPT / Codex 多账号代理网关
</p>
Codex Proxy RS 为多个 Codex 账号提供统一的 Responses API。客户端使用同一个服务地址和 API Key;账号调度、并发限制、额度检查、故障换号和会话续接都在服务端完成。
<p align="center">
<img src="https://img.shields.io/badge/Rust-1.95-000?style=flat-square&logo=rust" />
<img src="https://img.shields.io/badge/Axum-0.8-2f7d95?style=flat-square" />
<img src="https://img.shields.io/badge/Vue-3.5-42b883?style=flat-square&logo=vuedotjs&logoColor=white" />
<img src="https://img.shields.io/badge/Vite-8-646cff?style=flat-square&logo=vite&logoColor=white" />
<img src="https://img.shields.io/badge/SQLite-local-003b57?style=flat-square&logo=sqlite" />
<img src="https://img.shields.io/badge/License-MIT-blue?style=flat-square" />
</p>
## 功能
## 概览
- 兼容 `/v1/responses`、SSE 流式响应、Review、Compact 和模型查询。
- 支持 HTTP/SSE 与 WebSocket 上游连接。
- 提供 `smart`、额度重置优先、轮询和粘性四种账号调度策略。
- 单个账号发生额度、认证、模型或传输故障时,继续使用账号池中的其他可用账号。
- 使用会话亲和和受限历史重放处理 `previous_response_not_found` 与跨账号续接。
- 为每个账号派生稳定且隔离的 installation、session、thread、turn 和 cache 标识。
- 提供账号、API Key、模型映射、运行参数、用量、错误记录和系统更新管理界面。
- 使用 PostgreSQL 保存权威业务数据,使用 Redis 保存会话、租约、模型快照和会话亲和数据。
- 内置日志轮转、遥测保留期清理和 PostgreSQL/Redis 健康检查。
Codex Proxy RS 是一个单进程服务:
代理只补充账号凭据、上游请求头和账号作用域身份,并在必要时执行会话恢复;其余请求与响应尽量保持上游语义。
- Rust/Axum 后端提供 OpenAI 兼容 `/v1/*` 代理、管理端 API、SQLite 持久化和静态前端托管。
- Vue 管理端用于账号导入、API Key、用量统计、请求记录、运行参数、模型映射和系统更新。
- 运行数据默认收敛到仓库 `.runtime/`,Docker 也使用同一套宿主机目录做 bind mount。
## 快速部署
## 环境
需要 Docker Engine 和 Docker Compose Plugin。下面的 Compose 默认使用:
- Rust 1.95
- Node 24 或兼容版本,pnpm 11(只在开发前端或构建镜像时需要)
- Docker / Docker Compose(部署或验证镜像时需要)
- 应用镜像:`ghcr.io/zyycn/codex-proxy-rs:latest`
- PostgreSQL 18
- Redis 8
- 管理端地址:`http://127.0.0.1:8080`
## 本地运行
本地直接运行后端时,配置文件由 `CPR_CONFIG_FILE` 指定;未指定时读取当前工作目录下的 `config.yaml`。
### 1. 获取项目
```bash
cp deploy/config.example.yaml .runtime/config.yaml
git clone https://github.com/zyycn/codex-proxy-rs.git
cd codex-proxy-rs
```
如果直接用本地二进制而不是 Docker,确保 `.runtime/config.yaml` 里的路径是宿主机路径:
```yaml
database:
url: 'sqlite://.runtime/data/codex-proxy-rs.sqlite'
logging:
directory: .runtime/logs
admin:
default_password: '<set-a-long-random-password>'
```
启动:
```bash
mkdir -p .runtime/data .runtime/logs
CPR_CONFIG_FILE=.runtime/config.yaml cargo run --manifest-path backend/Cargo.toml
```
服务默认监听 `http://0.0.0.0:8080`。
## Docker 部署
Compose 默认把宿主机 `.runtime` 映射进容器:
- `.runtime/config.yaml` -> `/app/config.yaml`
- `.runtime/data` -> `/app/data`
- `.runtime/logs` -> `/app/logs`
初始化配置:
### 2. 创建配置
```bash
mkdir -p .runtime/data .runtime/logs
cp deploy/config.example.yaml .runtime/config.yaml
# 编辑 .runtime/config.yaml,设置 admin.default_password
cp deploy/.env.example deploy/.env
chmod 0600 deploy/.env
```
Docker 配置文件里路径保持容器路径:
编辑 `deploy/.env`:
```yaml
database:
url: 'sqlite:///app/data/codex-proxy-rs.sqlite'
logging:
directory: /app/logs
```dotenv
CPR_ADMIN_DEFAULT_PASSWORD='<管理员密码,至少 12 个字符>'
CPR_POSTGRES_PASSWORD='<PostgreSQL 密码>'
CPR_REDIS_PASSWORD='<Redis 密码>'
```
构建并启动:
`deploy/.env` 已被 Git 忽略。应用会安全地把 PostgreSQL 和 Redis 密码写入运行时连接 URL,不需要把真实密码填入 `.runtime/config.yaml`。
Linux 宿主机需要保证容器用户 `10001:10001` 可以读写数据和日志目录:
```bash
docker compose -f deploy/docker-compose.yml build
docker compose -f deploy/docker-compose.yml up -d
sudo chown -R "$(id -u):10001" .runtime/data .runtime/logs
chmod 0770 .runtime/data .runtime/logs
sudo chown "$(id -u):10001" .runtime/config.yaml
chmod 0640 .runtime/config.yaml
```
默认只绑定宿主机 `127.0.0.1:8080`。如需覆盖路径:
### 3. 启动
```bash
CPR_CONFIG_FILE=/path/to/config.yaml \
CPR_DATA_DIR=/path/to/data \
CPR_LOG_DIR=/path/to/logs \
docker compose -f deploy/docker-compose.yml up -d
docker compose --env-file deploy/.env -f deploy/docker-compose.yml config
docker compose --env-file deploy/.env -f deploy/docker-compose.yml pull
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d --no-build
```
## API
检查服务:
客户端 API Key 在管理端创建。OpenAI 兼容接口通过 `Authorization: Bearer <key>` 鉴权。
```bash
docker compose --env-file deploy/.env -f deploy/docker-compose.yml ps
curl -i http://127.0.0.1:8080/healthz
```
| 端点 | 说明 |
| --- | --- |
| `POST /v1/responses` | OpenAI Responses 兼容接口 |
| `GET /v1/models` | 模型列表 |
| `GET /v1/models/{id}` | 模型详情 |
| `GET /v1/models/catalog` | 管理端可见模型目录 |
| `/api/admin/*` | 管理端 API |
`/healthz` 正常时返回 `204 No Content`。随后打开 <http://127.0.0.1:8080>:
示例:
- 用户名:`admin@cpr.local`
- 密码:`deploy/.env` 中的 `CPR_ADMIN_DEFAULT_PASSWORD`
首次登录后导入账号,在“API 密钥”页面创建客户端 Key。
### 本地构建镜像
需要验证当前源码时,不拉取 GHCR 镜像,直接构建:
```bash
docker compose --env-file deploy/.env -f deploy/docker-compose.yml build codex-proxy-rs
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d
```
## 客户端接入
OpenAI 兼容客户端填写:
- Base URL:`http://127.0.0.1:8080/v1`
- API Key:管理端创建的 `sk_...` 客户端 Key
Responses 请求示例:
```bash
curl http://127.0.0.1:8080/v1/responses \
-H 'Authorization: Bearer <your-api-key>' \
-H 'Authorization: Bearer <client-api-key>' \
-H 'Content-Type: application/json' \
-d '{"model":"gpt-5.5","input":"Say hello","stream":true}'
-d '{
"model": "<model-id>",
"input": "Reply with pong.",
"stream": false
}'
```
可用接口:
| 路由 | 说明 |
| --- | --- |
| `POST /v1/responses` | Responses 非流式或 SSE 流式请求 |
| `POST /v1/responses/review` | Review 请求 |
| `POST /v1/responses/compact` | Compact 请求 |
| `GET /v1/models` | 模型列表 |
| `GET /v1/models/catalog` | 模型目录 |
| `GET /v1/models/{model_id}` | 模型详情 |
| `GET /v1/models/{model_id}/info` | 模型运行信息 |
Compose 默认只监听宿主机 `127.0.0.1`。需要从其他设备访问时,应在前面配置 HTTPS 反向代理,不要直接暴露 PostgreSQL 和 Redis 端口。
## 管理端
生产环境前端由后端直接托管,不需要单独 Node 服务。开发前端时:
管理端提供以下页面:
- Dashboard:请求趋势、服务状态、账号概览和最近请求。
- 账号:导入、导出、OAuth、连接测试、quota 和 token 刷新。
- API 密钥:创建、停用、删除和客户端接入信息。
- 用量:请求记录、token、延迟、模型分布和运维错误。
- 设置:模型别名、账号调度策略、单账号并发、请求间隔和刷新参数。
- 系统:版本检查、更新、重启和回滚。
管理端登录会话保存在 Redis,并按 `admin.session_ttl_minutes` 自动过期。
## 配置
启动配置位于 `.runtime/config.yaml`,模板见 [deploy/config.example.yaml](deploy/config.example.yaml)。
| 配置段 | 用途 |
| --- | --- |
| `server` | HTTP 监听地址和端口 |
| `api` | Codex 上游地址 |
| `database`、`redis` | PostgreSQL 与 Redis 连接 |
| `quota` | quota 刷新周期和耗尽账号过滤 |
| `tls` | 上游 HTTP 协议偏好 |
| `ws_pool` | WebSocket 连接池与首事件超时 |
| `fingerprint` | Codex Desktop 请求指纹 |
| `admin` | 默认管理员和登录会话 TTL |
| `logging` | 日志级别、目录、大小和保留期 |
| `telemetry` | 请求事实记录开关 |
模型别名、调度策略、单账号并发、请求间隔和刷新参数由管理端保存到 PostgreSQL,可在运行中更新。请求事实、运维错误和聚合时间桶由后台任务按各自保留期自动清理。
## 数据与升级
| 数据 | 默认位置 |
| --- | --- |
| 账号、API Key、设置、用量和错误记录 | Compose 命名卷 `postgres-data` |
| 管理会话、租约、模型快照和会话亲和 | Compose 命名卷 `redis-data` |
| 身份派生密钥和更新状态 | `.runtime/data` |
| 文件日志 | `.runtime/logs` |
| 启动配置 | `.runtime/config.yaml` |
| Docker 密钥 | `deploy/.env` |
普通升级只替换应用镜像:
```bash
pnpm --dir frontend install
pnpm --dir frontend dev
docker compose --env-file deploy/.env -f deploy/docker-compose.yml pull codex-proxy-rs
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d --no-build
```
生产构建:
更新镜像不会删除数据库卷,但仍应在升级前备份 PostgreSQL 和 `.runtime/data`。
不要使用 `docker compose down -v` 做普通升级,该命令会删除 PostgreSQL 与 Redis 命名卷。
`.runtime/data/identity_hmac_secret` 必须长期保留。丢失该文件会改变全部账号的 installation ID 和账号作用域身份。
## 查看状态
```bash
pnpm --dir frontend build
CPR_CONFIG_FILE=.runtime/config.yaml cargo run --manifest-path backend/Cargo.toml
# 容器状态
docker compose --env-file deploy/.env -f deploy/docker-compose.yml ps
# 应用日志
docker compose --env-file deploy/.env -f deploy/docker-compose.yml logs -f codex-proxy-rs
# PostgreSQL 和 Redis 日志
docker compose --env-file deploy/.env -f deploy/docker-compose.yml logs postgres redis
# 展开并检查最终 Compose 配置
docker compose --env-file deploy/.env -f deploy/docker-compose.yml config
```
管理端主要功能:
`/healthz` 返回 `503` 时,先检查 PostgreSQL 和 Redis 的健康状态。容器反复重启时,再检查 `.runtime/config.yaml`、`deploy/.env` 和 `.runtime/data`、`.runtime/logs` 的权限。
- 账号导入、OAuth 授权、连接测试、quota 刷新、token 刷新。
- 客户端 API Key 管理。
- 请求记录、Token 用量、模型用量和账号额度。
- 运行参数、模型别名、账号选择策略。
- 版本检查、在线更新、更新日志、重启和回滚。
## 架构
## 配置与运行数据
配置文件只承载启动必需项。账号、API Key、模型映射、账号选择策略和多数运行参数由管理端写入 SQLite。
`.runtime` 约定:
```text
.runtime/config.yaml # 本地或 Docker 启动配置,按实际运行环境写路径
.runtime/data/codex-proxy-rs.sqlite # SQLite 数据库
.runtime/data/installation_id # 上游 installation id
.runtime/data/update-state.json # 在线更新状态
.runtime/data/update.lock # 在线更新锁
.runtime/logs/ # 日志
```
## 发布与更新
版本信息在 `release/version.yaml`,发布平台在 `release/platforms.yaml`。
发布命令:
```bash
release/publish 0.1.5
```
该命令会更新版本文件、生成版本提交、创建 `v<version>` tag 并推送,随后 GitHub Actions 构建 Release 归档和 GHCR 多平台镜像。
在线更新由主服务处理,管理端调用 `/api/admin/system/*`:
- 检查 GitHub Release。
- 下载匹配当前平台的归档和 `checksums.txt`。
- 校验 checksum。
- 替换二进制和 `web/dist`。
- 重启时,Docker 模式依赖 `restart: unless-stopped` 拉起新容器进程;非 Docker 模式会先安排新进程延迟启动,再关闭当前进程。
## 项目结构
```text
backend/ Rust/Axum 后端、SQLite migration、集成测试
frontend/ Vue 3 管理端
deploy/ Dockerfile、Compose、部署配置模板
docs/ 架构、审计和维护文档
release/ 版本、平台和发布脚本
skills/ 项目本地 Codex skill
```
## 开发检查
```bash
cargo fmt --manifest-path backend/Cargo.toml --check
cargo clippy --manifest-path backend/Cargo.toml --all-targets --all-features --locked -- -D warnings
cargo test --manifest-path backend/Cargo.toml --test main --locked
pnpm --dir frontend build
docker compose -f deploy/docker-compose.yml config
```
模块边界、请求链路、换号条件、previous response 恢复、存储归属和后台任务见 [架构说明](docs/architecture.md)。
## License
+219 -503
View File
File diff suppressed because it is too large Load Diff
+16 -8
View File
@@ -4,24 +4,28 @@ edition = "2021"
license = "MIT"
publish = false
rust-version = "1.95"
build = "build/build.rs"
[dependencies]
argon2 = "0.5.3"
async-trait = "0.1.89"
axum = "0.8.9"
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
base64 = "0.22.1"
bytes = "1.11.1"
chrono = { version = "0.4.45", features = ["serde", "clock"] }
config = "0.15.23"
dirs = "5.0.1"
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
dirs = "6.0.0"
flate2 = "1.1.9"
futures = "0.3.32"
hex = "0.4.3"
hmac = "0.12.1"
indexmap = { version = "2.0", features = ["serde"] }
ipnet = "2.12.0"
percent-encoding = "2.3.2"
rand = "0.10.1"
# Enables password_hash::rand_core::OsRng for argon2 0.5.
rand_core = { version = "0.6.4", features = ["getrandom"] }
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
# Pinned to the Codex Desktop 26.707.41301 HTTP stack; re-audit a newer Desktop before upgrading.
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "gzip", "brotli", "zstd", "deflate", "http2"] }
rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] }
rustls-native-certs = "0.8.4"
@@ -31,11 +35,12 @@ semver = "1.0.27"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.150", features = ["preserve_order"] }
sha2 = "0.10.9"
sqlx = { version = "0.9.0", features = ["runtime-tokio", "tls-rustls", "sqlite", "chrono", "uuid"] }
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "sqlite-bundled", "chrono", "json"] }
subtle = "2.6.1"
tar = "0.4.44"
thiserror = "2.0.18"
tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "signal", "fs", "time", "sync", "net", "test-util"] }
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "signal", "fs", "time", "sync", "net"] }
tokio-util = { version = "0.7.18", features = ["rt"] }
tokio-tungstenite = { version = "0.28.0", features = ["proxy", "rustls-tls-native-roots"] }
tower = { version = "0.5.3", features = ["util"] }
tower-http = { version = "0.6.11", features = ["trace", "cors", "request-id", "timeout", "fs"] }
@@ -43,15 +48,18 @@ tracing = "0.1.44"
tracing-appender = "0.2.5"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
tungstenite = { version = "0.27.0", features = ["deflate", "proxy"] }
url = "2.5.8"
uuid = { version = "1.23.3", features = ["v4", "v7", "serde"] }
[dev-dependencies]
sqlx = { version = "0.9.0", default-features = false, features = ["derive"] }
tempfile = "3.27.0"
tokio = { version = "1.52.3", default-features = false, features = ["test-util"] }
wiremock = "0.6.5"
[patch.crates-io]
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "132f5b39c862e3a970f731d709608b3e6276d5f6" }
tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev = "9200079d3b54a1ff51072e24d81fd354f085156f" }
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186" }
tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev = "4fffad30fe373adbdcffab9545e9e9bf4f2fc19f" }
[lints.rust]
future_incompatible = { level = "warn", priority = -1 }
-5
View File
@@ -1,5 +0,0 @@
//! 管理端账号 HTTP 模块。
pub(crate) mod quota_view;
pub mod routes;
pub mod service;
File diff suppressed because it is too large Load Diff
@@ -1,11 +0,0 @@
use super::AdminAccountService;
impl AdminAccountService {
pub(crate) async fn usage_cookie_header(&self, account_id: &str) -> Option<String> {
self.cookies
.cookie_header_for_request(account_id, "chatgpt.com", "/codex/usage")
.await
.ok()
.flatten()
}
}
-161
View File
@@ -1,161 +0,0 @@
//! 管理端账号业务服务。
mod contracts;
mod cookies;
mod exporting;
mod importing;
mod lifecycle;
pub(crate) mod oauth;
mod quota;
mod testing;
use std::sync::Arc as StdArc;
use chrono::{DateTime, Utc};
use crate::{
upstream::accounts::{
cookies::SqliteCookieStore,
pool::RuntimeAccountPoolService,
store::SqliteAccountStore,
token_refresh::{RuntimeRefreshPolicy, TokenRefresher},
},
upstream::models::service::ModelService,
upstream::transport::CodexBackendClient,
};
pub use contracts::{
AdminAccountError, AdminAccountHealthCheck, AdminAccountMetadata, AdminAccountRefreshOutcome,
AdminAccountRefreshResult, AdminAccountUpdate, BatchDeleteAccounts, ExportedAccounts,
ImportedAccounts, OAuthAuthorizeResult, OAuthExchangeInput,
};
#[derive(Clone)]
pub struct AdminAccountService {
pub store: SqliteAccountStore,
pub(crate) cookies: SqliteCookieStore,
pub(crate) codex: StdArc<CodexBackendClient>,
pub(crate) models: StdArc<ModelService>,
pub(crate) account_pool: StdArc<RuntimeAccountPoolService>,
pub(crate) token_refresher: StdArc<dyn TokenRefresher>,
pub(crate) oauth: oauth::AccountOAuthService,
pub(crate) refresh_policy: RuntimeRefreshPolicy,
pub(crate) installation_id: Option<String>,
}
pub struct AdminAccountServiceParts {
pub store: SqliteAccountStore,
pub cookies: SqliteCookieStore,
pub codex: StdArc<CodexBackendClient>,
pub models: StdArc<ModelService>,
pub account_pool: StdArc<RuntimeAccountPoolService>,
pub token_refresher: StdArc<dyn TokenRefresher>,
pub oauth: oauth::AccountOAuthService,
pub refresh_policy: RuntimeRefreshPolicy,
pub installation_id: Option<String>,
}
impl AdminAccountService {
pub fn new(parts: AdminAccountServiceParts) -> Self {
Self {
store: parts.store,
cookies: parts.cookies,
codex: parts.codex,
models: parts.models,
account_pool: parts.account_pool,
token_refresher: parts.token_refresher,
oauth: parts.oauth,
refresh_policy: parts.refresh_policy,
installation_id: parts.installation_id,
}
}
pub(crate) fn next_refresh_at_for_expires_at(
&self,
account_id: &str,
expires_at: DateTime<chrono::Utc>,
) -> DateTime<chrono::Utc> {
crate::upstream::accounts::token_refresh::jittered_refresh_at(
account_id,
expires_at,
self.refresh_policy.refresh_margin_seconds(),
)
}
async fn refresh_tokens_from_refresh_token(
&self,
refresh_token: &str,
) -> Result<contracts::RefreshedAccountTokens, AdminAccountError> {
let token_pair = self
.token_refresher
.refresh(refresh_token)
.await
.map_err(AdminAccountError::RefreshTokenExchange)?;
let access_token = crate::upstream::accounts::importing::normalize_nonempty(Some(
crate::upstream::accounts::importing::normalize_bearer_token(&token_pair.access_token),
))
.ok_or(AdminAccountError::TokenRequired)?;
let claims = crate::upstream::accounts::token_refresh::manual_account_claims(
&access_token,
Utc::now(),
)
.map_err(AdminAccountError::InvalidToken)?;
Ok(contracts::RefreshedAccountTokens {
access_token,
refresh_token: token_pair.refresh_token,
claims,
})
}
pub(crate) async fn sync_account_pool(
&self,
account_id: &str,
) -> Result<(), AdminAccountError> {
self.account_pool
.sync_account_from_repository(account_id)
.await
.map(|_| ())
.map_err(|_| AdminAccountError::SyncAccountPool)
}
pub(crate) async fn sync_account_pool_best_effort(&self, account_id: &str, operation: &str) {
if let Err(error) = self
.account_pool
.sync_account_from_repository(account_id)
.await
{
tracing::warn!(
account_id,
operation,
error = %error,
"failed to sync runtime account pool after admin account update"
);
}
}
pub(crate) async fn evict_account_websocket_pool(&self, account_id: &str) {
self.codex.evict_websocket_account(account_id).await;
match self.store.get(account_id).await {
Ok(Some(account)) => {
if let Some(upstream_account_id) = account
.account_id
.as_deref()
.filter(|value| *value != account_id)
{
self.codex
.evict_websocket_account(upstream_account_id)
.await;
}
}
Ok(None) => {}
Err(error) => {
tracing::warn!(
account_id,
error = %error,
"failed to inspect account while evicting websocket pool"
);
}
}
}
}
@@ -1,52 +0,0 @@
use secrecy::ExposeSecret;
use super::{contracts::AdminAccountError, AdminAccountService};
impl AdminAccountService {
pub async fn account_quota(
&self,
account_id: &str,
) -> Result<serde_json::Value, AdminAccountError> {
let stored = self
.store
.get(account_id)
.await
.map_err(|_| AdminAccountError::NotFound)?
.ok_or(AdminAccountError::NotFound)?;
let request_id = uuid::Uuid::new_v4().to_string();
let token = stored.access_token.expose_secret().to_string();
let cookie_header = self.usage_cookie_header(account_id).await;
let context = crate::upstream::transport::CodexRequestContext {
access_token: &token,
account_id: stored.account_id.as_deref(),
request_id: &request_id,
turn_state: None,
turn_metadata: None,
beta_features: None,
include_timing_metrics: None,
version: None,
codex_window_id: None,
parent_thread_id: None,
cookie_header: cookie_header.as_deref(),
installation_id: self.installation_id.as_deref(),
session_id: None,
};
let raw = self
.codex
.fetch_usage(context)
.await
.map_err(|e| AdminAccountError::FetchQuota(e.to_string()))?;
let normalized = crate::upstream::accounts::quota::quota_from_usage(&raw);
if let Ok(json_str) = serde_json::to_string(&normalized) {
if matches!(
self.store.update_quota_json(account_id, &json_str).await,
Ok(true)
) {
self.sync_account_pool_best_effort(account_id, "account quota refresh")
.await;
}
}
Ok(serde_json::json!({ "quota": normalized, "raw": raw }))
}
}
@@ -1,382 +0,0 @@
use std::{convert::Infallible, pin::Pin};
use bytes::Bytes;
use futures::{Stream, StreamExt};
use secrecy::ExposeSecret;
use serde_json::{json, Value};
use tokio::sync::mpsc;
use crate::upstream::{
accounts::{model::Account, store::StoredAccount},
models::{info::CodexModelInfo, service::ModelRefreshPlanAccount},
protocol::{
responses::CodexResponsesRequest,
sse::{encode_sse_event, parse_sse_events, SseEvent},
},
transport::CodexRequestContext,
};
use super::{contracts::AdminAccountError, AdminAccountService};
pub(super) type AccountTestStream = Pin<Box<dyn Stream<Item = Result<Bytes, Infallible>> + Send>>;
#[derive(Debug, Clone, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountModelOption {
pub id: String,
pub label: String,
}
impl AdminAccountService {
pub async fn account_models(
&self,
account_id: &str,
) -> Result<Vec<AccountModelOption>, AdminAccountError> {
let account = self
.store
.get(account_id)
.await
.map_err(|_| AdminAccountError::Inspect)?
.ok_or(AdminAccountError::NotFound)?;
let plan_type = account_plan_type(&account);
let mut models = self.models.catalog().await.models_for_plan(&plan_type);
if models.is_empty() {
self.refresh_account_plan_models(&account, &plan_type).await;
models = self.models.catalog().await.models_for_plan(&plan_type);
}
let models = models.iter().map(account_model_option).collect::<Vec<_>>();
if models.is_empty() {
return Err(AdminAccountError::NoModels);
}
Ok(models)
}
async fn refresh_account_plan_models(&self, account: &StoredAccount, plan_type: &str) {
let request_id = uuid::Uuid::new_v4().to_string();
let plan_account = ModelRefreshPlanAccount {
plan_type: plan_type.to_string(),
account: stored_account_to_model_refresh_account(account.clone()),
};
if let Err(error) = self
.models
.refresh_backend_models_with_installation_id(
&[plan_account],
&request_id,
self.installation_id.as_deref(),
)
.await
{
tracing::warn!(
account_id = %account.id,
plan_type,
error = %error,
"failed to refresh account plan models"
);
}
}
pub async fn test_connection_stream(
&self,
account_id: &str,
model: String,
) -> Result<AccountTestStream, AdminAccountError> {
let account = self
.store
.get(account_id)
.await
.map_err(|_| AdminAccountError::Inspect)?
.ok_or(AdminAccountError::NotFound)?;
let token = account.access_token.expose_secret().to_string();
let upstream_account_id = account.account_id.clone();
let cookie_header = self
.cookies
.cookie_header_for_request(&account.id, "chatgpt.com", "/codex/responses")
.await
.ok()
.flatten();
let installation_id = self.installation_id.clone();
let codex = self.codex.clone();
let request_id = uuid::Uuid::new_v4().to_string();
let (tx, rx) = mpsc::channel::<Bytes>(16);
tokio::spawn(async move {
send_test_event(
&tx,
json!({
"type": "test_start",
"model": model.clone(),
"text": "正在连接 Codex Responses"
}),
)
.await;
let request = test_responses_request(model);
send_test_event(
&tx,
json!({
"type": "request",
"payload": serde_json::to_value(&request).unwrap_or_else(|_| json!({}))
}),
)
.await;
let context = CodexRequestContext {
access_token: &token,
account_id: upstream_account_id.as_deref(),
request_id: &request_id,
turn_state: None,
turn_metadata: None,
beta_features: None,
include_timing_metrics: None,
version: None,
codex_window_id: None,
parent_thread_id: None,
cookie_header: cookie_header.as_deref(),
installation_id: installation_id.as_deref(),
session_id: None,
};
match codex.create_response_stream(&request, context).await {
Ok(response) => process_upstream_test_stream(response.body, tx).await,
Err(error) => {
send_test_event(
&tx,
json!({
"type": "error",
"error": error.to_string()
}),
)
.await;
}
}
});
let stream = futures::stream::unfold(rx, |mut rx| async {
rx.recv()
.await
.map(|bytes| (Ok::<Bytes, Infallible>(bytes), rx))
});
Ok(Box::pin(stream))
}
}
fn account_model_option(model: &CodexModelInfo) -> AccountModelOption {
AccountModelOption {
id: model.id.clone(),
label: model.display_name.clone(),
}
}
fn account_plan_type(account: &StoredAccount) -> String {
account
.plan_type
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("default")
.to_string()
}
fn stored_account_to_model_refresh_account(stored: StoredAccount) -> Account {
Account {
id: stored.id,
email: stored.email,
account_id: stored.account_id,
user_id: stored.user_id,
label: stored.label,
plan_type: stored.plan_type,
access_token: stored.access_token.expose_secret().to_string(),
refresh_token: stored
.refresh_token
.map(|token| token.expose_secret().to_string()),
access_token_expires_at: stored.access_token_expires_at,
next_refresh_at: stored.next_refresh_at,
status: stored.status,
quota_limit_reached: false,
quota_verify_required: false,
quota_cooldown_until: None,
cloudflare_cooldown_until: None,
request_count: 0,
empty_response_count: 0,
image_input_tokens: 0,
image_output_tokens: 0,
image_request_count: 0,
image_request_failed_count: 0,
window_request_count: 0,
window_input_tokens: 0,
window_output_tokens: 0,
window_cached_tokens: 0,
window_image_input_tokens: 0,
window_image_output_tokens: 0,
window_image_request_count: 0,
window_image_request_failed_count: 0,
window_started_at: None,
window_reset_at: None,
limit_window_seconds: None,
added_at: stored.added_at,
last_used_at: None,
}
}
fn test_responses_request(model: String) -> CodexResponsesRequest {
let mut request = CodexResponsesRequest::new_http_sse(
model,
"You are checking whether this Codex account can answer. Reply with ok.",
vec![json!({
"role": "user",
"content": [
{
"type": "input_text",
"text": "hi"
}
]
})],
);
request.force_http_sse = true;
request
}
async fn process_upstream_test_stream(
mut body: crate::upstream::transport::CodexBackendSseStream,
tx: mpsc::Sender<Bytes>,
) {
let mut buffer = String::new();
while let Some(chunk) = body.next().await {
let chunk = match chunk {
Ok(bytes) => bytes,
Err(error) => {
send_test_event(&tx, json!({ "type": "error", "error": error.to_string() })).await;
return;
}
};
buffer.push_str(&String::from_utf8_lossy(&chunk));
while let Some(frame) = take_sse_frame(&mut buffer) {
if process_sse_frame(&tx, &frame).await {
return;
}
}
}
if !buffer.trim().is_empty() && process_sse_frame(&tx, &buffer).await {
return;
}
send_test_event(
&tx,
json!({
"type": "error",
"error": "Stream ended before response.completed"
}),
)
.await;
}
async fn process_sse_frame(tx: &mpsc::Sender<Bytes>, frame: &str) -> bool {
let events = match parse_sse_events(frame) {
Ok(events) => events,
Err(error) => {
send_test_event(
tx,
json!({
"type": "error",
"error": error.to_string()
}),
)
.await;
return true;
}
};
for event in events {
if process_sse_event(tx, &event).await {
return true;
}
}
false
}
async fn process_sse_event(tx: &mpsc::Sender<Bytes>, event: &SseEvent) -> bool {
let value: Value = match serde_json::from_str(&event.data) {
Ok(value) => value,
Err(_) => return false,
};
match value.get("type").and_then(Value::as_str) {
Some("response.output_text.delta") => {
if let Some(delta) = value.get("delta").and_then(Value::as_str) {
if !delta.is_empty() {
send_test_event(tx, json!({ "type": "content", "text": delta })).await;
}
}
false
}
Some("response.completed" | "response.done") => {
send_test_event(tx, json!({ "type": "test_complete", "success": true })).await;
true
}
Some("response.failed") => {
send_test_event(
tx,
json!({
"type": "error",
"error": response_failure_message(&value)
}),
)
.await;
true
}
Some("error") => {
send_test_event(
tx,
json!({
"type": "error",
"error": error_event_message(&value)
}),
)
.await;
true
}
_ => false,
}
}
fn response_failure_message(value: &Value) -> String {
value
.pointer("/response/error/message")
.or_else(|| value.pointer("/error/message"))
.or_else(|| value.get("message"))
.and_then(Value::as_str)
.filter(|message| !message.trim().is_empty())
.unwrap_or("OpenAI response failed")
.to_string()
}
fn error_event_message(value: &Value) -> String {
value
.pointer("/error/message")
.or_else(|| value.get("message"))
.and_then(Value::as_str)
.filter(|message| !message.trim().is_empty())
.unwrap_or("Unknown upstream error")
.to_string()
}
fn take_sse_frame(buffer: &mut String) -> Option<String> {
let index = buffer.find("\n\n").or_else(|| buffer.find("\r\n\r\n"))?;
let delimiter_len = if buffer[index..].starts_with("\r\n\r\n") {
4
} else {
2
};
let frame = buffer[..index + delimiter_len].to_string();
buffer.drain(..index + delimiter_len);
Some(frame)
}
async fn send_test_event(tx: &mpsc::Sender<Bytes>, event: Value) {
let _ = tx
.send(Bytes::from(encode_sse_event("", &event.to_string())))
.await;
}
-4
View File
@@ -1,4 +0,0 @@
//! 管理端认证与本地会话模块。
pub mod service;
pub mod session;
-341
View File
@@ -1,341 +0,0 @@
//! 管理员会话存储与认证领域逻辑。
use std::{collections::HashMap, sync::Arc};
use chrono::{DateTime, Duration, Utc};
use sqlx::SqlitePool;
use thiserror::Error;
use tokio::sync::Mutex;
use uuid::Uuid;
use crate::infra::identity::{hash_admin_password, verify_admin_password};
const LOGIN_FAILURE_WINDOW: Duration = Duration::minutes(15);
const LOGIN_LOCK_DURATION: Duration = Duration::minutes(15);
const LOGIN_MAX_FAILURES: u32 = 5;
/// 管理员登录成功后的会话。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AdminLoginSession {
/// 会话 ID。
pub session_id: String,
/// 过期时间。
pub expires_at: DateTime<Utc>,
}
/// 管理员会话错误。
#[derive(Debug, Error)]
pub enum AdminSessionError {
/// 校验管理员会话失败。
#[error("failed to validate admin session")]
Validate,
/// 计算管理员密码哈希失败。
#[error("failed to hash admin password")]
HashPassword,
/// 创建默认管理员失败。
#[error("failed to create default admin user")]
CreateAdmin,
/// 读取管理员用户失败。
#[error("failed to load admin user")]
LoadAdmin,
/// 校验管理员密码失败。
#[error("failed to verify admin password")]
VerifyPassword,
/// 创建管理员会话失败。
#[error("failed to create admin session")]
CreateSession,
/// 删除管理员会话失败。
#[error("failed to delete admin session")]
DeleteSession,
/// 管理员登录尝试过于频繁。
#[error("too many admin login attempts")]
LoginThrottled,
}
/// 管理员会话服务。
#[derive(Clone)]
pub struct AdminSessionService {
store: SqliteAdminSessionStore,
default_username: String,
session_ttl_minutes: u64,
login_failures: Arc<Mutex<HashMap<String, LoginFailureState>>>,
}
#[derive(Debug, Clone)]
struct LoginFailureState {
failures: u32,
first_failed_at: DateTime<Utc>,
locked_until: Option<DateTime<Utc>>,
}
impl AdminSessionService {
/// 构造服务。
pub fn new(
store: SqliteAdminSessionStore,
default_username: String,
session_ttl_minutes: u64,
) -> Self {
Self {
store,
default_username,
session_ttl_minutes,
login_failures: Arc::new(Mutex::new(HashMap::new())),
}
}
/// 校验管理员会话。
pub async fn validate(&self, session_id: Option<&str>) -> Result<bool, AdminSessionError> {
let Some(session_id) = session_id else {
return Ok(false);
};
self.store
.validate_session(session_id)
.await
.map_err(|_| AdminSessionError::Validate)
}
/// 确保默认管理员存在。
pub async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminSessionError> {
let password_hash =
hash_admin_password(password).map_err(|_| AdminSessionError::HashPassword)?;
self.store
.ensure_default_admin(&password_hash)
.await
.map_err(|_| AdminSessionError::CreateAdmin)
}
/// 登录并创建管理员会话。
pub async fn login(
&self,
source: &str,
username: Option<&str>,
password: &str,
) -> Result<Option<AdminLoginSession>, AdminSessionError> {
let source = login_source_key(source);
self.ensure_login_allowed(&source).await?;
let username = username.unwrap_or(&self.default_username);
if username != self.default_username.as_str() {
self.record_failed_login(&source).await;
return Ok(None);
}
let Some(admin) = self
.store
.load_first_admin()
.await
.map_err(|_| AdminSessionError::LoadAdmin)?
else {
self.record_failed_login(&source).await;
return Ok(None);
};
let password_matches = verify_admin_password(password, &admin.password_hash)
.map_err(|_| AdminSessionError::VerifyPassword)?;
if !password_matches {
self.record_failed_login(&source).await;
return Ok(None);
}
self.clear_failed_login(&source).await;
let session_id = format!("sess_{}", Uuid::new_v4().simple());
let ttl_minutes = self.session_ttl_minutes.min(i64::MAX as u64) as i64;
let expires_at = Utc::now() + Duration::minutes(ttl_minutes);
self.store
.create_session(&session_id, &admin.id, expires_at)
.await
.map_err(|_| AdminSessionError::CreateSession)?;
Ok(Some(AdminLoginSession {
session_id,
expires_at,
}))
}
/// 删除管理员会话。
pub async fn delete_session(&self, session_id: &str) -> Result<bool, AdminSessionError> {
self.store
.delete_session(session_id)
.await
.map_err(|_| AdminSessionError::DeleteSession)
}
async fn ensure_login_allowed(&self, source: &str) -> Result<(), AdminSessionError> {
let now = Utc::now();
let mut failures = self.login_failures.lock().await;
retain_active_login_failures(&mut failures, now);
if failures
.get(source)
.and_then(|state| state.locked_until)
.is_some_and(|locked_until| locked_until > now)
{
return Err(AdminSessionError::LoginThrottled);
}
Ok(())
}
async fn record_failed_login(&self, source: &str) {
let now = Utc::now();
let mut failures = self.login_failures.lock().await;
retain_active_login_failures(&mut failures, now);
let state = failures
.entry(source.to_string())
.or_insert_with(|| LoginFailureState {
failures: 0,
first_failed_at: now,
locked_until: None,
});
if now - state.first_failed_at > LOGIN_FAILURE_WINDOW {
state.failures = 0;
state.first_failed_at = now;
state.locked_until = None;
}
state.failures += 1;
if state.failures >= LOGIN_MAX_FAILURES {
state.locked_until = Some(now + LOGIN_LOCK_DURATION);
}
}
async fn clear_failed_login(&self, source: &str) {
self.login_failures.lock().await.remove(source);
}
}
fn login_source_key(source: &str) -> String {
let source = source.trim();
if source.is_empty() {
"unknown".to_string()
} else {
source.to_string()
}
}
fn retain_active_login_failures(
failures: &mut HashMap<String, LoginFailureState>,
now: DateTime<Utc>,
) {
failures.retain(|_, state| {
state
.locked_until
.is_some_and(|locked_until| locked_until > now)
|| now - state.first_failed_at <= LOGIN_FAILURE_WINDOW
});
}
// ---------------------------------------------------------------------------
// 已持久化的管理用户
// ---------------------------------------------------------------------------
/// 已持久化的管理员用户。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoredAdminUser {
/// 管理员用户 ID。
pub id: String,
/// 管理员密码哈希。
pub password_hash: String,
}
// ---------------------------------------------------------------------------
// SQLite 管理员会话存储
// ---------------------------------------------------------------------------
/// SQLite 管理员会话存储结果。
pub type SqliteAdminSessionStoreResult<T> = Result<T, sqlx::Error>;
/// SQLite 管理员会话存储。
#[derive(Clone)]
pub struct SqliteAdminSessionStore {
pool: SqlitePool,
}
impl SqliteAdminSessionStore {
/// 构造存储。
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
}
/// 如果还没有管理员用户,则创建默认管理员。
pub async fn ensure_default_admin(
&self,
password_hash: &str,
) -> SqliteAdminSessionStoreResult<bool> {
let existing: (i64,) = sqlx::query_as("select count(*) from admin_users")
.fetch_one(&self.pool)
.await?;
if existing.0 > 0 {
return Ok(false);
}
let now = Utc::now().to_rfc3339();
sqlx::query(
"insert into admin_users (id, password_hash, created_at, updated_at) values (?, ?, ?, ?)",
)
.bind("admin_1")
.bind(password_hash)
.bind(&now)
.bind(&now)
.execute(&self.pool)
.await?;
Ok(true)
}
/// 读取第一个管理员用户。
pub async fn load_first_admin(&self) -> SqliteAdminSessionStoreResult<Option<StoredAdminUser>> {
let row = sqlx::query_as::<_, (String, String)>(
"select id, password_hash from admin_users order by created_at asc, id asc limit 1",
)
.fetch_optional(&self.pool)
.await?;
Ok(row.map(|(id, password_hash)| StoredAdminUser { id, password_hash }))
}
/// 创建管理员会话。
pub async fn create_session(
&self,
session_id: &str,
user_id: &str,
expires_at: DateTime<Utc>,
) -> SqliteAdminSessionStoreResult<()> {
let now = Utc::now().to_rfc3339();
sqlx::query(
"insert into admin_sessions (id, user_id, expires_at, created_at) values (?, ?, ?, ?)",
)
.bind(session_id)
.bind(user_id)
.bind(expires_at.to_rfc3339())
.bind(now)
.execute(&self.pool)
.await?;
Ok(())
}
/// 校验管理员会话是否存在且未过期。
pub async fn validate_session(&self, session_id: &str) -> SqliteAdminSessionStoreResult<bool> {
let now = Utc::now().to_rfc3339();
let row = sqlx::query("select 1 from admin_sessions where id = ? and expires_at > ?")
.bind(session_id)
.bind(now)
.fetch_optional(&self.pool)
.await?;
Ok(row.is_some())
}
/// 删除已经过期的管理员会话。
pub async fn cleanup_expired_sessions(
&self,
now: DateTime<Utc>,
) -> SqliteAdminSessionStoreResult<u64> {
let result = sqlx::query("delete from admin_sessions where expires_at <= ?")
.bind(now.to_rfc3339())
.execute(&self.pool)
.await?;
Ok(result.rows_affected())
}
/// 删除指定的管理员会话。
pub async fn delete_session(&self, session_id: &str) -> SqliteAdminSessionStoreResult<bool> {
let result = sqlx::query("delete from admin_sessions where id = ?")
.bind(session_id)
.execute(&self.pool)
.await?;
Ok(result.rows_affected() > 0)
}
}
-4
View File
@@ -1,4 +0,0 @@
//! 管理端 v1 接口访问 Key 模块。
pub mod routes;
pub mod service;
-698
View File
@@ -1,698 +0,0 @@
//! 客户端 API Key 端口定义、业务服务与 SQLite 存储适配器。
use std::{
collections::{BTreeMap, HashMap},
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex, RwLock,
},
time::Duration,
};
use async_trait::async_trait;
use chrono::Utc;
use sqlx::{Row, SqlitePool};
use thiserror::Error;
use tokio::time::sleep;
use uuid::Uuid;
use crate::infra::{
identity::generate_client_api_key,
json::{decode_cursor, encode_cursor, Page},
};
// ---------------------------------------------------------------------------
// 端口定义
// ---------------------------------------------------------------------------
/// 客户端 API Key 存储错误。
#[derive(Debug, Error)]
pub enum ClientKeyStoreError {
/// 底层存储失败。
#[error("client key store operation failed: {message}")]
OperationFailed {
/// 错误说明。
message: String,
},
}
/// 客户端 API Key 存储结果类型。
pub type ClientKeyStoreResult<T> = Result<T, ClientKeyStoreError>;
/// 提供客户端 API Key 验证能力的端口。
#[async_trait]
pub trait ClientKeyStore: Send + Sync + 'static {
/// 验证明文客户端 API Key,并在成功时记录使用时间。
async fn verify_and_touch(&self, plaintext: &str) -> ClientKeyStoreResult<bool>;
}
const CLIENT_KEY_LAST_USED_FLUSH_DELAY: Duration = Duration::from_secs(1);
// ---------------------------------------------------------------------------
// 业务服务
// ---------------------------------------------------------------------------
/// 客户端 API Key 服务。
#[derive(Clone)]
pub struct ClientKeyService {
store: Arc<RuntimeClientKeyStore>,
}
impl ClientKeyService {
/// 构造服务。
pub fn new(store: Arc<RuntimeClientKeyStore>) -> Self {
Self { store }
}
/// 验证客户端 API Key。
pub async fn verify(&self, plaintext: &str) -> ClientKeyStoreResult<bool> {
if !plaintext.starts_with("sk_") {
return Ok(false);
}
self.store.verify_and_touch(plaintext).await
}
/// 重新加载运行时内存鉴权表。
pub async fn reload_from_store(&self) -> ClientKeyStoreResult<()> {
self.store.reload_from_store().await
}
/// 立即刷写待持久化的最近使用时间。
pub async fn flush_pending_last_used(&self) {
self.store.flush_pending_last_used().await;
}
}
// ---------------------------------------------------------------------------
// 管理端业务服务
// ---------------------------------------------------------------------------
/// 管理端客户端 API Key 服务。
#[derive(Clone)]
pub struct AdminClientKeyService {
store: SqliteClientKeyStore,
runtime: Arc<RuntimeClientKeyStore>,
}
impl AdminClientKeyService {
/// 构造服务。
pub fn new(store: SqliteClientKeyStore, runtime: Arc<RuntimeClientKeyStore>) -> Self {
Self { store, runtime }
}
/// 创建客户端 API Key。
pub async fn create(&self, name: &str) -> Result<AdminClientApiKey, AdminClientKeyError> {
let name = name.trim();
if name.is_empty() {
return Err(AdminClientKeyError::EmptyName);
}
self.store
.create(name)
.await
.inspect(|created| self.runtime.upsert_stored(created))
.map(AdminClientApiKey::from)
.map_err(|_| AdminClientKeyError::Create)
}
/// 分页列出客户端 API Key。
pub async fn list(
&self,
cursor: Option<String>,
limit: u32,
) -> Result<Page<AdminClientApiKey>, AdminClientKeyError> {
let page = self
.store
.list(cursor, limit)
.await
.map_err(|_| AdminClientKeyError::List)?;
Ok(Page {
items: page
.items
.into_iter()
.map(AdminClientApiKey::from)
.collect(),
next_cursor: page.next_cursor,
})
}
/// 按 ID 读取客户端 API Key 元数据。
pub async fn get(
&self,
key_id: &str,
) -> Result<Option<AdminClientApiKey>, AdminClientKeyError> {
self.store
.get(key_id)
.await
.map(|key| key.map(AdminClientApiKey::from))
.map_err(|_| AdminClientKeyError::List)
}
/// 更新客户端 API Key 标签。
pub async fn update_label(
&self,
key_id: &str,
label: Option<String>,
) -> Result<Option<AdminClientApiKey>, AdminClientKeyError> {
if label.as_ref().is_some_and(|l| l.chars().count() > 64) {
return Err(AdminClientKeyError::LabelTooLong);
}
self.store
.set_label(key_id, label)
.await
.map(|key| key.map(AdminClientApiKey::from))
.map_err(|_| AdminClientKeyError::UpdateLabel)
}
/// 更新客户端 API Key 启用状态。
pub async fn update_status(
&self,
key_id: &str,
status: &str,
) -> Result<bool, AdminClientKeyError> {
let enabled = parse_client_key_status(status)?;
let updated = self
.store
.set_enabled(key_id, enabled)
.await
.map_err(|_| AdminClientKeyError::UpdateStatus)?;
if updated {
if enabled {
let Some(key) = self
.store
.get(key_id)
.await
.map_err(|_| AdminClientKeyError::UpdateStatus)?
else {
return Ok(false);
};
self.runtime.upsert_stored(&key);
} else {
self.runtime.remove_by_id(key_id);
}
}
Ok(updated)
}
/// 批量删除客户端 API Key。
pub async fn batch_delete(
&self,
ids: Vec<String>,
) -> Result<BatchDeleteClientApiKeys, AdminClientKeyError> {
if ids.is_empty() {
return Err(AdminClientKeyError::EmptyIds);
}
let mut deleted = 0u32;
let mut not_found = Vec::new();
for id in ids {
match self.store.delete(&id).await {
Ok(true) => {
self.runtime.remove_by_id(&id);
deleted += 1;
}
Ok(false) => not_found.push(id),
Err(_) => return Err(AdminClientKeyError::Delete),
}
}
Ok(BatchDeleteClientApiKeys { deleted, not_found })
}
}
/// 管理端可见的客户端 API Key。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AdminClientApiKey {
pub id: String,
pub name: String,
pub label: Option<String>,
pub prefix: String,
pub key: String,
pub enabled: bool,
pub created_at: String,
pub last_used_at: Option<String>,
}
/// 客户端 API Key 批量删除结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BatchDeleteClientApiKeys {
pub deleted: u32,
pub not_found: Vec<String>,
}
/// 管理端客户端 API Key 错误。
#[derive(Debug, Error)]
pub enum AdminClientKeyError {
#[error("failed to list client API keys")]
List,
#[error("failed to create client API key")]
Create,
#[error("failed to delete client API key")]
Delete,
#[error("failed to update client API key label")]
UpdateLabel,
#[error("failed to update client API key status")]
UpdateStatus,
#[error("unsupported client API key status: {0}")]
InvalidStatus(String),
#[error("client API key name is required")]
EmptyName,
#[error("client API key ids are required")]
EmptyIds,
#[error("client API key label must be 64 characters or fewer")]
LabelTooLong,
}
fn parse_client_key_status(status: &str) -> Result<bool, AdminClientKeyError> {
match status.trim().to_ascii_lowercase().as_str() {
"active" => Ok(true),
"disabled" => Ok(false),
other => Err(AdminClientKeyError::InvalidStatus(other.to_string())),
}
}
impl From<StoredClientApiKey> for AdminClientApiKey {
fn from(key: StoredClientApiKey) -> Self {
Self {
id: key.id,
name: key.name,
label: key.label,
prefix: key.prefix,
key: key.key,
enabled: key.enabled,
created_at: key.created_at,
last_used_at: key.last_used_at,
}
}
}
// ---------------------------------------------------------------------------
// SQLite 存储适配器
// ---------------------------------------------------------------------------
/// SQLite 客户端 API Key 存储错误。
#[derive(Debug, Error)]
pub enum SqliteClientKeyStoreError {
/// 数据库错误。
#[error("sqlite client key database error: {0}")]
Database(#[from] sqlx::Error),
/// 分页游标无效。
#[error("invalid client key pagination cursor")]
InvalidCursor,
}
/// 已持久化的客户端 API Key 元数据。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoredClientApiKey {
/// API Key 记录 ID。
pub id: String,
/// API Key 名称。
pub name: String,
/// 管理员可见标签。
pub label: Option<String>,
/// 明文 API Key 的短前缀。
pub prefix: String,
/// 管理端可复制的完整 API Key。
pub key: String,
/// 是否允许用于 `/v1` 认证。
pub enabled: bool,
/// 创建时间。
pub created_at: String,
/// 最近一次成功使用时间。
pub last_used_at: Option<String>,
}
/// SQLite 客户端 API Key 存储。
#[derive(Clone)]
pub struct SqliteClientKeyStore {
pool: SqlitePool,
}
impl SqliteClientKeyStore {
/// 构造存储适配器。
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
}
/// 暴露底层连接池,供集成测试和运行时组合层复用。
pub fn pool(&self) -> &SqlitePool {
&self.pool
}
/// 读取所有已启用客户端 API Key,用于运行时内存鉴权表初始化。
pub async fn list_enabled(&self) -> Result<Vec<StoredClientApiKey>, SqliteClientKeyStoreError> {
let rows = sqlx::query(
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where enabled = 1 order by created_at asc, id asc",
)
.fetch_all(&self.pool)
.await?;
Ok(rows
.into_iter()
.map(|row| Self::key_from_row(&row))
.collect())
}
/// 创建新的本地客户端 API Key。
pub async fn create(
&self,
name: &str,
) -> Result<StoredClientApiKey, SqliteClientKeyStoreError> {
let generated = generate_client_api_key();
let id = format!("key_{}", Uuid::new_v4().simple());
let now = Utc::now().to_rfc3339();
sqlx::query(
"insert into client_api_keys (id, name, label, prefix, key, enabled, created_at, last_used_at) values (?, ?, null, ?, ?, 1, ?, null)",
)
.bind(&id)
.bind(name)
.bind(&generated.prefix)
.bind(&generated.key)
.bind(&now)
.execute(&self.pool)
.await?;
Ok(StoredClientApiKey {
id,
name: name.to_string(),
label: None,
prefix: generated.prefix,
key: generated.key,
enabled: true,
created_at: now,
last_used_at: None,
})
}
/// 按创建时间倒序分页列出客户端 API Key。
pub async fn list(
&self,
cursor: Option<String>,
limit: u32,
) -> Result<Page<StoredClientApiKey>, SqliteClientKeyStoreError> {
let fetch_limit = i64::from(limit) + 1;
let rows = if let Some(cursor) = cursor {
let (created_at, id) =
decode_cursor(&cursor).ok_or(SqliteClientKeyStoreError::InvalidCursor)?;
sqlx::query(
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where created_at < ? or (created_at = ? and id < ?) order by created_at desc, id desc limit ?",
)
.bind(&created_at)
.bind(created_at)
.bind(id)
.bind(fetch_limit)
.fetch_all(&self.pool)
.await?
} else {
sqlx::query(
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys order by created_at desc, id desc limit ?",
)
.bind(fetch_limit)
.fetch_all(&self.pool)
.await?
};
let has_next = rows.len() > limit as usize;
let take_count = rows.len().min(limit as usize);
let items = rows
.into_iter()
.take(take_count)
.map(|row| Self::key_from_row(&row))
.collect::<Vec<_>>();
let next_cursor = if has_next {
items
.last()
.map(|key| encode_cursor(&key.created_at, &key.id))
} else {
None
};
Ok(Page { items, next_cursor })
}
/// 按 ID 读取客户端 API Key 元数据。
pub async fn get(
&self,
id: &str,
) -> Result<Option<StoredClientApiKey>, SqliteClientKeyStoreError> {
let row = sqlx::query(
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where id = ?",
)
.bind(id)
.fetch_optional(&self.pool)
.await?;
Ok(row.map(|row| Self::key_from_row(&row)))
}
/// 更新客户端 API Key 启用状态。
pub async fn set_enabled(
&self,
id: &str,
enabled: bool,
) -> Result<bool, SqliteClientKeyStoreError> {
let result = sqlx::query("update client_api_keys set enabled = ? where id = ?")
.bind(if enabled { 1_i64 } else { 0_i64 })
.bind(id)
.execute(&self.pool)
.await?;
Ok(result.rows_affected() > 0)
}
/// 更新客户端 API Key 标签并返回最新元数据。
pub async fn set_label(
&self,
id: &str,
label: Option<String>,
) -> Result<Option<StoredClientApiKey>, SqliteClientKeyStoreError> {
let result = sqlx::query("update client_api_keys set label = ? where id = ?")
.bind(label)
.bind(id)
.execute(&self.pool)
.await?;
if result.rows_affected() == 0 {
return Ok(None);
}
self.get(id).await
}
/// 删除客户端 API Key。
pub async fn delete(&self, id: &str) -> Result<bool, SqliteClientKeyStoreError> {
let result = sqlx::query("delete from client_api_keys where id = ?")
.bind(id)
.execute(&self.pool)
.await?;
Ok(result.rows_affected() > 0)
}
/// 更新最近使用时间。
pub async fn update_last_used_at(
&self,
id: &str,
last_used_at: &str,
) -> Result<bool, SqliteClientKeyStoreError> {
let result = sqlx::query("update client_api_keys set last_used_at = ? where id = ?")
.bind(last_used_at)
.bind(id)
.execute(&self.pool)
.await?;
Ok(result.rows_affected() > 0)
}
fn key_from_row(row: &sqlx::sqlite::SqliteRow) -> StoredClientApiKey {
StoredClientApiKey {
id: row.get("id"),
name: row.get("name"),
label: row.get("label"),
prefix: row.get("prefix"),
key: row.get("key"),
enabled: row.get::<i64, _>("enabled") != 0,
created_at: row.get("created_at"),
last_used_at: row.get("last_used_at"),
}
}
}
#[async_trait]
impl ClientKeyStore for RuntimeClientKeyStore {
async fn verify_and_touch(&self, plaintext: &str) -> ClientKeyStoreResult<bool> {
let Some(id) = self.lookup_key_id(plaintext) else {
return Ok(false);
};
self.queue_last_used_touch(id);
Ok(true)
}
}
/// 运行时客户端 API Key 内存鉴权表。
#[derive(Clone)]
pub struct RuntimeClientKeyStore {
store: SqliteClientKeyStore,
cache: Arc<RwLock<RuntimeClientKeyCache>>,
pending_last_used: Arc<Mutex<BTreeMap<String, String>>>,
flush_scheduled: Arc<AtomicBool>,
flush_delay: Duration,
}
#[derive(Default)]
struct RuntimeClientKeyCache {
keys_by_plaintext: HashMap<String, String>,
plaintext_by_id: HashMap<String, String>,
}
impl RuntimeClientKeyStore {
/// 构造运行时内存鉴权表。
pub fn new(store: SqliteClientKeyStore) -> Self {
Self {
store,
cache: Arc::new(RwLock::new(RuntimeClientKeyCache::default())),
pending_last_used: Arc::new(Mutex::new(BTreeMap::new())),
flush_scheduled: Arc::new(AtomicBool::new(false)),
flush_delay: CLIENT_KEY_LAST_USED_FLUSH_DELAY,
}
}
/// 从 SQLite 加载当前启用的客户端 API Key。
pub async fn reload_from_store(&self) -> ClientKeyStoreResult<()> {
let keys = self
.store
.list_enabled()
.await
.map_err(|error| map_client_key_store_error(&error))?;
let mut keys_by_plaintext = HashMap::with_capacity(keys.len());
let mut plaintext_by_id = HashMap::with_capacity(keys.len());
for key in keys {
keys_by_plaintext.insert(key.key.clone(), key.id.clone());
plaintext_by_id.insert(key.id, key.key);
}
*self
.cache
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = RuntimeClientKeyCache {
keys_by_plaintext,
plaintext_by_id,
};
Ok(())
}
fn lookup_key_id(&self, plaintext: &str) -> Option<String> {
self.cache
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.keys_by_plaintext
.get(plaintext)
.cloned()
}
fn upsert_stored(&self, key: &StoredClientApiKey) {
if key.enabled {
self.upsert_enabled_key(&key.id, &key.key);
} else {
self.remove_by_id(&key.id);
}
}
fn upsert_enabled_key(&self, id: &str, plaintext: &str) {
let mut cache = self
.cache
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(old_plaintext) = cache.plaintext_by_id.remove(id) {
cache.keys_by_plaintext.remove(&old_plaintext);
}
cache
.keys_by_plaintext
.insert(plaintext.to_string(), id.to_string());
cache
.plaintext_by_id
.insert(id.to_string(), plaintext.to_string());
}
fn remove_by_id(&self, id: &str) {
let mut cache = self
.cache
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(plaintext) = cache.plaintext_by_id.remove(id) {
cache.keys_by_plaintext.remove(&plaintext);
}
drop(cache);
self.pending_last_used
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(id);
}
fn queue_last_used_touch(&self, id: String) {
self.pending_last_used
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(id, Utc::now().to_rfc3339());
self.schedule_flush();
}
fn schedule_flush(&self) {
if self
.flush_scheduled
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return;
}
let runtime = self.clone();
tokio::spawn(async move {
sleep(runtime.flush_delay).await;
runtime.flush_pending_last_used().await;
});
}
/// 立即刷写待持久化的最近使用时间。
pub async fn flush_pending_last_used(&self) {
let updates = self.take_pending_last_used();
let mut failed = BTreeMap::new();
for (id, last_used_at) in updates {
if let Err(error) = self.store.update_last_used_at(&id, &last_used_at).await {
tracing::warn!(
key_id = %id,
error = %error,
"failed to flush client api key last_used_at"
);
failed.insert(id, last_used_at);
}
}
if !failed.is_empty() {
self.pending_last_used
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.extend(failed);
}
self.flush_scheduled.store(false, Ordering::Release);
if !self
.pending_last_used
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty()
{
self.schedule_flush();
}
}
fn take_pending_last_used(&self) -> BTreeMap<String, String> {
std::mem::take(
&mut *self
.pending_last_used
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
)
}
}
fn map_client_key_store_error(error: &SqliteClientKeyStoreError) -> ClientKeyStoreError {
ClientKeyStoreError::OperationFailed {
message: error.to_string(),
}
}
-19
View File
@@ -1,19 +0,0 @@
//! 管理端 HTTP API。
/// 账号 CRUD 处理器。
pub mod accounts;
/// 管理端账号认证状态与本地管理员会话处理器。
pub mod auth;
/// v1 接口访问 Key 处理器。
pub mod keys;
/// 管理面日志、用量与诊断统计。
pub mod monitoring;
/// 响应封装。
pub mod response;
/// 管理端路由。
pub mod router;
/// 设置处理器。
pub mod settings;
/// 系统版本与在线更新处理器。
pub mod system;
pub(crate) mod update_payload;
@@ -1,382 +0,0 @@
//! SQLite 用量存储。
use chrono::{DateTime, Utc};
use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite, SqlitePool};
use thiserror::Error;
use crate::infra::{
json::{decode_cursor, encode_cursor, Page},
time::{
parse_optional_rfc3339_utc as parse_optional_rfc3339, parse_rfc3339_utc as parse_rfc3339,
},
};
const LIST_USAGE_SELECT_SQL: &str = r"
select
au.account_id,
a.email,
a.label,
a.plan_type,
au.request_count,
au.empty_response_count,
au.input_tokens,
au.output_tokens,
au.cached_tokens,
au.reasoning_tokens,
au.total_tokens,
au.image_input_tokens,
au.image_output_tokens,
au.image_request_count,
au.image_request_failed_count,
au.window_request_count,
au.window_input_tokens,
au.window_output_tokens,
au.window_cached_tokens,
au.window_started_at,
au.window_reset_at,
au.limit_window_seconds,
au.last_used_at
from account_usage au
left join accounts a on a.id = au.account_id";
const USAGE_SUMMARY_SQL: &str = r"
select
count(*) as account_count,
coalesce(sum(request_count), 0) as request_count,
coalesce(sum(empty_response_count), 0) as empty_response_count,
coalesce(sum(input_tokens), 0) as input_tokens,
coalesce(sum(output_tokens), 0) as output_tokens,
coalesce(sum(cached_tokens), 0) as cached_tokens,
coalesce(sum(reasoning_tokens), 0) as reasoning_tokens,
coalesce(sum(total_tokens), 0) as total_tokens,
coalesce(sum(image_input_tokens), 0) as image_input_tokens,
coalesce(sum(image_output_tokens), 0) as image_output_tokens,
coalesce(sum(image_request_count), 0) as image_request_count,
coalesce(sum(image_request_failed_count), 0) as image_request_failed_count
from account_usage";
const LIST_USAGE_TIME_BUCKETS_SQL: &str = r"
select
bucket_start,
model,
service_tier,
coalesce(sum(request_count), 0) as request_count,
coalesce(sum(error_count), 0) as error_count,
coalesce(sum(input_tokens), 0) as input_tokens,
coalesce(sum(output_tokens), 0) as output_tokens,
coalesce(sum(cached_tokens), 0) as cached_tokens,
coalesce(sum(first_token_latency_sum), 0) as first_token_latency_sum,
coalesce(sum(first_token_latency_count), 0) as first_token_latency_count,
coalesce(sum(latency_sum), 0) as latency_sum,
coalesce(sum(latency_count), 0) as latency_count,
coalesce(max(max_latency_ms), 0) as max_latency_ms,
coalesce(min(nullif(min_latency_ms, 0)), 0) as min_latency_ms
from usage_time_buckets
where bucket_start >= ? and bucket_start <= ?
group by bucket_start, model, service_tier
order by bucket_start asc, model asc, service_tier asc";
/// SQLite 用量存储错误。
#[derive(Debug, Error)]
pub enum SqliteUsageStoreError {
/// 数据库错误。
#[error("sqlite usage store database error: {0}")]
Database(#[from] sqlx::Error),
/// 时间格式错误。
#[error("sqlite usage store timestamp error: {0}")]
Timestamp(#[from] chrono::ParseError),
/// 分页游标非法。
#[error("invalid usage pagination cursor")]
InvalidCursor,
}
/// SQLite 用量存储结果。
pub type SqliteUsageStoreResult<T> = Result<T, SqliteUsageStoreError>;
/// 账号用量列表记录。
#[derive(Debug, Clone)]
pub struct UsageListRecord {
/// 账号 ID。
pub account_id: String,
/// 邮箱。
pub email: Option<String>,
/// 标签。
pub label: Option<String>,
/// 计划类型。
pub plan_type: Option<String>,
/// 历史请求总数。
pub request_count: i64,
/// 历史空响应次数。
pub empty_response_count: i64,
/// 累计输入 token。
pub input_tokens: i64,
/// 累计输出 token。
pub output_tokens: i64,
/// 累计缓存 token。
pub cached_tokens: i64,
/// 累计 reasoning token。
pub reasoning_tokens: i64,
/// 累计总 token。
pub total_tokens: i64,
/// 累计图片输入 token。
pub image_input_tokens: i64,
/// 累计图片输出 token。
pub image_output_tokens: i64,
/// 累计图片请求成功次数。
pub image_request_count: i64,
/// 累计图片请求失败次数。
pub image_request_failed_count: i64,
/// 当前额度窗口请求数。
pub window_request_count: i64,
/// 当前额度窗口输入 token。
pub window_input_tokens: i64,
/// 当前额度窗口输出 token。
pub window_output_tokens: i64,
/// 当前额度窗口缓存 token。
pub window_cached_tokens: i64,
/// 当前额度窗口开始时间。
pub window_started_at: Option<DateTime<Utc>>,
/// 当前额度窗口重置时间。
pub window_reset_at: Option<DateTime<Utc>>,
/// 当前额度窗口大小(秒)。
pub limit_window_seconds: Option<u64>,
/// 最近使用时间。
pub last_used_at: Option<DateTime<Utc>>,
}
/// 账号用量汇总。
#[derive(Debug, Clone)]
pub struct UsageSummary {
/// 账号数。
pub account_count: i64,
/// 总请求数。
pub request_count: i64,
/// 总空响应数。
pub empty_response_count: i64,
/// 总输入 token。
pub input_tokens: i64,
/// 总输出 token。
pub output_tokens: i64,
/// 总缓存 token。
pub cached_tokens: i64,
/// 总 reasoning token。
pub reasoning_tokens: i64,
/// 总 token。
pub total_tokens: i64,
/// 总图片输入 token。
pub image_input_tokens: i64,
/// 总图片输出 token。
pub image_output_tokens: i64,
/// 总图片请求成功数。
pub image_request_count: i64,
/// 总图片请求失败数。
pub image_request_failed_count: i64,
}
/// 时间桶聚合用量记录。
#[derive(Debug, Clone)]
pub struct UsageTimeBucketRecord {
/// 桶开始时间。
pub bucket_start: DateTime<Utc>,
/// 模型。
pub model: String,
/// 服务层级。
pub service_tier: Option<String>,
/// 请求数。
pub request_count: i64,
/// 错误数。
pub error_count: i64,
/// 输入 token。
pub input_tokens: i64,
/// 输出 token。
pub output_tokens: i64,
/// 缓存 token。
pub cached_tokens: i64,
/// 首 token 延迟总和。
pub first_token_latency_sum: i64,
/// 首 token 延迟样本数。
pub first_token_latency_count: i64,
/// 完成延迟总和。
pub latency_sum: i64,
/// 完成延迟样本数。
pub latency_count: i64,
/// 最大完成延迟。
pub max_latency_ms: i64,
/// 最小完成延迟。
pub min_latency_ms: i64,
}
/// SQLite 用量存储。
#[derive(Clone)]
pub struct SqliteUsageStore {
pool: SqlitePool,
}
impl SqliteUsageStore {
/// 构造存储。
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
}
/// 分页列出账号用量。
pub async fn list_usage(
&self,
cursor: Option<String>,
limit: u32,
) -> SqliteUsageStoreResult<Page<UsageListRecord>> {
let limit = limit.clamp(1, 200);
let fetch_limit = i64::from(limit) + 1;
let mut builder = QueryBuilder::<Sqlite>::new(LIST_USAGE_SELECT_SQL);
if let Some(cursor) = cursor {
let (last_used_at, account_id) =
decode_cursor(&cursor).ok_or(SqliteUsageStoreError::InvalidCursor)?;
builder.push("\nwhere au.last_used_at < ");
builder.push_bind(last_used_at.clone());
builder.push("\n or (au.last_used_at = ");
builder.push_bind(last_used_at);
builder.push(" and au.account_id < ");
builder.push_bind(account_id);
builder.push(")");
}
builder.push("\norder by au.last_used_at desc, au.account_id desc\nlimit ");
builder.push_bind(fetch_limit);
let rows = builder.build().fetch_all(&self.pool).await?;
Ok(to_page(&rows, limit))
}
/// 按账号 ID 批量读取账号用量。
pub async fn list_usage_by_account_ids(
&self,
account_ids: &[String],
) -> SqliteUsageStoreResult<Vec<UsageListRecord>> {
if account_ids.is_empty() {
return Ok(Vec::new());
}
let mut builder = QueryBuilder::<Sqlite>::new(LIST_USAGE_SELECT_SQL);
builder.push("\nwhere au.account_id in (");
let mut separated = builder.separated(", ");
for account_id in account_ids {
separated.push_bind(account_id);
}
separated.push_unseparated(")");
let rows = builder.build().fetch_all(&self.pool).await?;
rows.iter().map(usage_list_from_row).collect()
}
/// 汇总账号用量。
pub async fn usage_summary(&self) -> SqliteUsageStoreResult<UsageSummary> {
let row = sqlx::query(USAGE_SUMMARY_SQL).fetch_one(&self.pool).await?;
Ok(UsageSummary {
account_count: row.get("account_count"),
request_count: row.get("request_count"),
empty_response_count: row.get("empty_response_count"),
input_tokens: row.get("input_tokens"),
output_tokens: row.get("output_tokens"),
cached_tokens: row.get("cached_tokens"),
reasoning_tokens: row.get("reasoning_tokens"),
total_tokens: row.get("total_tokens"),
image_input_tokens: row.get("image_input_tokens"),
image_output_tokens: row.get("image_output_tokens"),
image_request_count: row.get("image_request_count"),
image_request_failed_count: row.get("image_request_failed_count"),
})
}
/// 列出指定时间范围内的时间桶聚合用量。
pub async fn list_time_buckets(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> SqliteUsageStoreResult<Vec<UsageTimeBucketRecord>> {
let rows = sqlx::query(LIST_USAGE_TIME_BUCKETS_SQL)
.bind(start.to_rfc3339())
.bind(end.to_rfc3339())
.fetch_all(&self.pool)
.await?;
rows.iter().map(usage_time_bucket_from_row).collect()
}
}
fn usage_list_from_row(row: &SqliteRow) -> SqliteUsageStoreResult<UsageListRecord> {
Ok(UsageListRecord {
account_id: row.get("account_id"),
email: row.get("email"),
label: row.get("label"),
plan_type: row.get("plan_type"),
request_count: row.get("request_count"),
empty_response_count: row.get("empty_response_count"),
input_tokens: row.get("input_tokens"),
output_tokens: row.get("output_tokens"),
cached_tokens: row.get("cached_tokens"),
reasoning_tokens: row.get("reasoning_tokens"),
total_tokens: row.get("total_tokens"),
image_input_tokens: row.get("image_input_tokens"),
image_output_tokens: row.get("image_output_tokens"),
image_request_count: row.get("image_request_count"),
image_request_failed_count: row.get("image_request_failed_count"),
window_request_count: row.get("window_request_count"),
window_input_tokens: row.get("window_input_tokens"),
window_output_tokens: row.get("window_output_tokens"),
window_cached_tokens: row.get("window_cached_tokens"),
window_started_at: parse_optional_rfc3339(
row.get::<Option<String>, _>("window_started_at").as_deref(),
)?,
window_reset_at: parse_optional_rfc3339(
row.get::<Option<String>, _>("window_reset_at").as_deref(),
)?,
limit_window_seconds: row
.get::<Option<i64>, _>("limit_window_seconds")
.and_then(|value| u64::try_from(value).ok()),
last_used_at: parse_optional_rfc3339(
row.get::<Option<String>, _>("last_used_at").as_deref(),
)?,
})
}
fn usage_time_bucket_from_row(row: &SqliteRow) -> SqliteUsageStoreResult<UsageTimeBucketRecord> {
let service_tier = row.get::<String, _>("service_tier").trim().to_string();
Ok(UsageTimeBucketRecord {
bucket_start: parse_rfc3339(row.get::<&str, _>("bucket_start"))?,
model: row.get("model"),
service_tier: (!service_tier.is_empty()).then_some(service_tier),
request_count: row.get("request_count"),
error_count: row.get("error_count"),
input_tokens: row.get("input_tokens"),
output_tokens: row.get("output_tokens"),
cached_tokens: row.get("cached_tokens"),
first_token_latency_sum: row.get("first_token_latency_sum"),
first_token_latency_count: row.get("first_token_latency_count"),
latency_sum: row.get("latency_sum"),
latency_count: row.get("latency_count"),
max_latency_ms: row.get("max_latency_ms"),
min_latency_ms: row.get("min_latency_ms"),
})
}
fn to_page(rows: &[SqliteRow], limit: u32) -> Page<UsageListRecord> {
let has_more = rows.len() > limit as usize;
let mut items = Vec::with_capacity(limit as usize);
let mut last_row: Option<&SqliteRow> = None;
for (i, row) in rows.iter().enumerate() {
if i >= limit as usize {
break;
}
if let Ok(item) = usage_list_from_row(row) {
items.push(item);
last_row = Some(row);
}
}
let next_cursor = if has_more {
last_row.map(|row| {
let ts: String = row.get("last_used_at");
let id: String = row.get("account_id");
encode_cursor(&ts, &id)
})
} else {
None
};
Page { items, next_cursor }
}
File diff suppressed because it is too large Load Diff
@@ -1,90 +0,0 @@
//! 诊断数据类型与聚合函数。
use serde::Serialize;
use crate::{upstream::accounts::pool::AccountCapacitySummary, upstream::fingerprint::Fingerprint};
/// 账号池摘要。
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountPoolDiagnostics {
/// 总数。
pub total: usize,
/// 活跃数。
pub active: usize,
/// 过期数。
pub expired: usize,
/// 配额耗尽数。
pub quota_exhausted: usize,
/// 刷新中数。
pub refreshing: usize,
/// 禁用数。
pub disabled: usize,
/// 封禁数。
pub banned: usize,
}
/// Account-pool capacity diagnostics.
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountCapacityDiagnostics {
/// Maximum concurrent slots per account.
pub max_concurrent_per_account: usize,
/// Total slots available across active accounts.
pub total_slots: usize,
/// Currently occupied slots.
pub used_slots: usize,
/// Currently available slots.
pub available_slots: usize,
}
/// Runtime fingerprint summary.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FingerprintDiagnostics {
/// Fingerprint source label.
pub source: &'static str,
/// Client originator.
pub originator: String,
/// App version.
pub app_version: String,
/// Build number.
pub build_number: String,
/// Platform name.
pub platform: String,
/// Architecture name.
pub arch: String,
/// Chromium major version.
pub chromium_version: String,
/// Expanded user-agent.
pub user_agent: String,
/// 指纹最后更新时间。
#[serde(skip_serializing_if = "Option::is_none")]
pub updated_at: Option<String>,
}
/// 构造 fingerprint 诊断数据。
pub fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics {
FingerprintDiagnostics {
source: "runtime",
originator: fingerprint.originator.clone(),
app_version: fingerprint.app_version.clone(),
build_number: fingerprint.build_number.clone(),
platform: fingerprint.platform.clone(),
arch: fingerprint.arch.clone(),
chromium_version: fingerprint.chromium_version.clone(),
user_agent: fingerprint.user_agent(),
updated_at: fingerprint.updated_at.clone(),
}
}
impl From<AccountCapacitySummary> for AccountCapacityDiagnostics {
fn from(summary: AccountCapacitySummary) -> Self {
Self {
max_concurrent_per_account: summary.max_concurrent_per_account,
total_slots: summary.total_slots,
used_slots: summary.used_slots,
available_slots: summary.available_slots,
}
}
}
-11
View File
@@ -1,11 +0,0 @@
//! 管理面监控模块:日志、用量统计和 Dashboard 聚合。
pub mod account_usage_service;
pub mod account_usage_store;
pub(crate) mod billing;
pub mod dashboard;
pub mod diagnostics;
pub mod usage_record_model;
pub mod usage_record_routes;
pub mod usage_record_service;
pub mod usage_record_store;
@@ -1,355 +0,0 @@
//! 管理端使用记录服务。
use std::{collections::HashMap, sync::Arc};
use chrono::{DateTime, Utc};
use thiserror::Error;
use crate::{
admin::monitoring::{
billing,
usage_record_model::metadata_service_tier,
usage_record_model::{UsageRecord, UsageRecordLevel},
usage_record_store::{
SqliteUsageRecordStore, UsageRecordBreakdown, UsageRecordEndpointSource,
UsageRecordFilter, UsageRecordModelSource, UsageRecordSummary, UsageRecordTrendPoint,
},
},
infra::json::{NumberedPage, Page},
};
/// 管理端使用记录查询过滤器。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AdminUsageRecordFilter {
/// 事件类别。
pub kind: Option<String>,
/// 事件等级。
pub level: Option<UsageRecordLevel>,
/// 请求 ID。
pub request_id: Option<String>,
/// 账号 ID。
pub account_id: Option<String>,
/// 路由。
pub route: Option<String>,
/// 模型。
pub model: Option<String>,
/// HTTP 状态码。
pub status_code: Option<i64>,
/// 上游传输方式。
pub transport: Option<String>,
/// 同一请求内的上游尝试序号。
pub attempt_index: Option<i64>,
/// 上游 HTTP 状态码。
pub upstream_status_code: Option<i64>,
/// 失败分类。
pub failure_class: Option<String>,
/// 上游响应 ID。
pub response_id: Option<String>,
/// 上游请求 ID。
pub upstream_request_id: Option<String>,
/// 搜索关键词。
pub search: Option<String>,
/// 起始时间。
pub start_time: Option<DateTime<Utc>>,
/// 结束时间。
pub end_time: Option<DateTime<Utc>>,
}
impl From<AdminUsageRecordFilter> for UsageRecordFilter {
fn from(filter: AdminUsageRecordFilter) -> Self {
Self {
kind: filter.kind,
level: filter.level,
request_id: filter.request_id,
account_id: filter.account_id,
route: filter.route,
model: filter.model,
status_code: filter.status_code,
transport: filter.transport,
attempt_index: filter.attempt_index,
upstream_status_code: filter.upstream_status_code,
failure_class: filter.failure_class,
response_id: filter.response_id,
upstream_request_id: filter.upstream_request_id,
search: filter.search,
start_time: filter.start_time,
end_time: filter.end_time,
}
}
}
/// 清空使用记录结果。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AdminClearUsageRecords {
/// 清理数量。
pub cleared: u64,
}
/// 管理端使用记录费用明细。
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct AdminUsageRecordCostDetails {
pub input_cost: f64,
pub output_cost: f64,
pub cache_read_cost: f64,
pub total_cost: f64,
pub original_cost: f64,
pub input_price_per_mtoken: f64,
pub output_price_per_mtoken: f64,
pub cache_read_price_per_mtoken: f64,
pub service_tier: Option<String>,
pub service_tier_display: String,
pub multiplier: f64,
}
/// 管理端使用记录错误。
#[derive(Debug, Error)]
pub enum AdminUsageRecordError {
/// 列表失败。
#[error("failed to list usage records")]
List,
/// 读取失败。
#[error("failed to get usage record")]
Get,
/// 清空失败。
#[error("failed to clear usage records")]
Clear,
/// 写入失败。
#[error("failed to append usage record")]
Append,
/// 裁剪失败。
#[error("failed to trim usage records")]
Trim,
/// 账号关联信息读取失败。
#[error("failed to load usage record accounts")]
Accounts,
}
/// 管理端使用记录服务。
#[derive(Clone)]
pub struct AdminUsageRecordService {
store: SqliteUsageRecordStore,
settings: Arc<tokio::sync::RwLock<AdminUsageRecordSettings>>,
}
#[derive(Debug, Clone, Copy)]
struct AdminUsageRecordSettings {
enabled: bool,
capacity: u32,
capture_body: bool,
}
impl AdminUsageRecordService {
/// 构造管理端使用记录服务。
pub fn new(
store: SqliteUsageRecordStore,
enabled: bool,
capacity: u32,
capture_body: bool,
) -> Self {
Self {
store,
settings: Arc::new(tokio::sync::RwLock::new(AdminUsageRecordSettings {
enabled,
capacity,
capture_body,
})),
}
}
/// 分页查询日志。
pub async fn list(
&self,
cursor: Option<String>,
limit: u32,
filter: AdminUsageRecordFilter,
) -> Result<Page<UsageRecord>, AdminUsageRecordError> {
self.store
.list(filter.into(), cursor, limit)
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 按页码查询日志。
pub async fn list_page(
&self,
page: u32,
page_size: u32,
filter: AdminUsageRecordFilter,
) -> Result<NumberedPage<UsageRecord>, AdminUsageRecordError> {
self.store
.list_page(filter.into(), page, page_size)
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 按 ID 读取日志。
pub async fn get(&self, id: &str) -> Result<Option<UsageRecord>, AdminUsageRecordError> {
self.store
.get(id)
.await
.map_err(|_| AdminUsageRecordError::Get)
}
/// 读取使用记录关联账号的邮箱映射。
pub async fn account_email_map(
&self,
items: &[UsageRecord],
) -> Result<HashMap<String, String>, AdminUsageRecordError> {
self.store
.account_email_map(items)
.await
.map_err(|_| AdminUsageRecordError::Accounts)
}
/// 汇总使用记录。
pub async fn summary(
&self,
filter: AdminUsageRecordFilter,
) -> Result<UsageRecordSummary, AdminUsageRecordError> {
self.store
.summary(filter.into())
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 按模型来源聚合使用记录分布。
pub async fn model_distribution(
&self,
filter: AdminUsageRecordFilter,
source: UsageRecordModelSource,
) -> Result<Vec<UsageRecordBreakdown>, AdminUsageRecordError> {
self.store
.model_distribution(filter.into(), source)
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 按端点来源聚合使用记录分布。
pub async fn endpoint_distribution(
&self,
filter: AdminUsageRecordFilter,
source: UsageRecordEndpointSource,
) -> Result<Vec<UsageRecordBreakdown>, AdminUsageRecordError> {
self.store
.endpoint_distribution(filter.into(), source)
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 聚合使用记录趋势。
pub async fn trend(
&self,
filter: AdminUsageRecordFilter,
) -> Result<Vec<UsageRecordTrendPoint>, AdminUsageRecordError> {
self.store
.trend(filter.into())
.await
.map_err(|_| AdminUsageRecordError::List)
}
/// 清空日志。
pub async fn clear(&self) -> Result<AdminClearUsageRecords, AdminUsageRecordError> {
self.store
.clear()
.await
.map(|cleared| AdminClearUsageRecords { cleared })
.map_err(|_| AdminUsageRecordError::Clear)
}
/// 记录使用记录。
pub async fn record(&self, event: UsageRecord) -> Result<(), AdminUsageRecordError> {
let settings = *self.settings.read().await;
if !settings.enabled && event.level != UsageRecordLevel::Error {
return Ok(());
}
self.append_with_settings(event, settings).await
}
async fn append_with_settings(
&self,
mut event: UsageRecord,
settings: AdminUsageRecordSettings,
) -> Result<(), AdminUsageRecordError> {
apply_capture_body_policy(&mut event, settings.capture_body);
self.store
.append(&event)
.await
.map_err(|_| AdminUsageRecordError::Append)?;
self.store
.trim_to_capacity(settings.capacity)
.await
.map_err(|_| AdminUsageRecordError::Trim)?;
Ok(())
}
}
fn apply_capture_body_policy(event: &mut UsageRecord, capture_body: bool) {
if capture_body {
return;
}
let Some(metadata) = event.metadata.as_object_mut() else {
return;
};
for key in [
"body",
"rawBody",
"requestBody",
"responseBody",
"upstreamBody",
] {
metadata.remove(key);
}
}
pub(crate) fn usage_record_cost_details(
record: &UsageRecord,
upstream_model: Option<&str>,
input_tokens: u64,
output_tokens: u64,
cached_tokens: u64,
) -> Option<AdminUsageRecordCostDetails> {
if input_tokens == 0 && output_tokens == 0 && cached_tokens == 0 {
return None;
}
let model = upstream_model
.or(record.model.as_deref())
.map(str::trim)
.filter(|value| !value.is_empty())?;
let service_tier = metadata_service_tier(&record.metadata);
let breakdown = billing::calculate_cost_breakdown(
input_tokens,
output_tokens,
cached_tokens,
model,
service_tier,
);
let original_cost = breakdown.input_cost + breakdown.output_cost + breakdown.cache_read_cost;
Some(AdminUsageRecordCostDetails {
input_cost: breakdown.input_cost,
output_cost: breakdown.output_cost,
cache_read_cost: breakdown.cache_read_cost,
total_cost: breakdown.total_cost,
original_cost,
input_price_per_mtoken: breakdown.input_price_per_mtoken,
output_price_per_mtoken: breakdown.output_price_per_mtoken,
cache_read_price_per_mtoken: breakdown.cache_read_price_per_mtoken,
service_tier: breakdown.service_tier.clone(),
service_tier_display: breakdown
.service_tier
.as_deref()
.map(format_service_tier)
.unwrap_or_else(|| "Default".to_string()),
multiplier: breakdown.tier_multiplier,
})
}
fn format_service_tier(value: &str) -> String {
match value {
"priority" | "fast" => "Fast".to_string(),
"flex" => "Flex".to_string(),
"default" => "Default".to_string(),
other => other.to_string(),
}
}
File diff suppressed because it is too large Load Diff
-3
View File
@@ -1,3 +0,0 @@
//! 管理端设置 HTTP 模块。
pub mod routes;
-5
View File
@@ -1,5 +0,0 @@
//! 系统版本与在线更新处理器。
pub mod routes;
mod state;
mod updater;
-5
View File
@@ -1,5 +0,0 @@
//! 管理端系统路由入口。
pub(crate) use super::updater::{
perform_update, restart, rollback, update_detail, update_event_stream, update_status, version,
};
-62
View File
@@ -1,62 +0,0 @@
//! 管理端通用更新请求解析。
use serde::Deserialize;
use serde_json::Value;
use crate::admin::response::AdminError;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct EditableUpdateRequest {
id: String,
label: Option<Option<String>>,
status: Option<String>,
}
pub(crate) struct ParsedEditableUpdate {
pub id: String,
pub label: Option<Option<String>>,
pub status: Option<String>,
}
pub(crate) struct EditableUpdateMessages<'a> {
pub object_required: &'a str,
pub invalid: &'a str,
pub empty_update: &'a str,
pub unknown_field_editable: bool,
}
pub(crate) fn parse_editable_update(
payload: &Value,
messages: EditableUpdateMessages<'_>,
) -> Result<ParsedEditableUpdate, AdminError> {
let object = payload
.as_object()
.ok_or_else(|| AdminError::bad_request(messages.object_required))?;
if messages.unknown_field_editable {
for field in object.keys() {
if !matches!(field.as_str(), "id" | "label" | "status") {
return Err(AdminError::bad_request(format!("{field} is not editable")));
}
}
}
let update = serde_json::from_value::<EditableUpdateRequest>(payload.clone())
.map_err(|_| AdminError::bad_request(messages.invalid))?;
if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) {
return Err(AdminError::bad_request(messages.invalid));
}
if update.label.is_none() && update.status.is_none() {
return Err(AdminError::bad_request(messages.empty_update));
}
Ok(ParsedEditableUpdate {
id: update.id,
label: update.label,
status: update.status,
})
}
fn is_blank(value: &str) -> bool {
value.trim().is_empty()
}
@@ -0,0 +1,24 @@
use super::*;
/// `GET /api/admin/accounts/export`
pub async fn export_accounts(
State(state): State<AppState>,
_auth: AdminAuth,
Query(query): Query<AccountExportQuery>,
) -> Result<impl IntoResponse, AdminError> {
let ids = account_export_ids(query.ids.as_deref());
if query.confirm.as_deref() != Some(ACCOUNT_EXPORT_CONFIRMATION) {
return Err(AdminError::bad_request(
"account export requires confirm=export_sensitive_accounts",
));
}
match state.services.admin_accounts.export(ids).await {
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(result),
)),
Err(error) => Err(account_error(&error)),
}
}
@@ -0,0 +1,20 @@
use super::*;
/// `POST /api/admin/accounts/import`
pub async fn import_accounts(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<Value>,
) -> Result<impl IntoResponse, AdminError> {
match state.services.admin_accounts.import(payload).await {
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountImportData {
imported: result.imported,
skipped: result.skipped,
source_format: result.source_format.to_string(),
}),
)),
Err(error) => Err(account_error(&error)),
}
}
@@ -0,0 +1,110 @@
use super::*;
/// `POST /api/admin/accounts/refresh`
pub async fn refresh_account(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<AccountActionRequest>,
) -> Result<impl IntoResponse, AdminError> {
let account_id = payload.id;
match state
.services
.admin_accounts
.refresh_account(&account_id)
.await
{
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountRefreshData::from(result)),
)),
Err(error) => Err(account_error(&error)),
}
}
/// `POST /api/admin/accounts/health-check`
pub async fn health_check_accounts(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<AccountHealthCheckRequest>,
) -> Result<impl IntoResponse, AdminError> {
let stagger_ms = payload.stagger_ms.unwrap_or(3_000);
if !(500..=30_000).contains(&stagger_ms) && stagger_ms != 0 {
return Err(AdminError::bad_request(
"stagger_ms must be between 500 and 30000",
));
}
if let Some(concurrency) = payload.concurrency {
if !(1..=10).contains(&concurrency) {
return Err(AdminError::bad_request(
"concurrency must be between 1 and 10",
));
}
}
match state
.services
.admin_accounts
.health_check_accounts(payload.ids, stagger_ms, payload.concurrency.unwrap_or(2))
.await
{
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountHealthCheckData::from(result)),
)),
Err(error) => Err(account_error(&error)),
}
}
/// `POST /api/admin/accounts/delete`
pub async fn batch_delete_accounts(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<BatchDeleteAccountsRequest>,
) -> Result<impl IntoResponse, AdminError> {
match state
.services
.admin_accounts
.batch_delete(payload.ids)
.await
{
Ok(result) => {
for account_id in &result.deleted_ids {
state
.services
.session_affinity
.forget_account(account_id)
.await;
}
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(BatchDeleteData {
deleted: result.deleted,
not_found: result.not_found,
}),
))
}
Err(error) => Err(account_error(&error)),
}
}
/// `POST /api/admin/accounts/update`
pub async fn update_account(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<Value>,
) -> Result<impl IntoResponse, AdminError> {
let ParsedAccountUpdate { id, update } = parse_account_update(&payload)?;
match state
.services
.admin_accounts
.update_account(&id, update)
.await
{
Ok(Some(account)) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountUpdateData::Account(Box::new(account.into()))),
)),
Ok(None) => Err(account_not_found()),
Err(error) => Err(account_error(&error)),
}
}
@@ -0,0 +1,572 @@
//! 账号管理 HTTP 处理器。
use axum::{
body::Body,
extract::{Query, State},
http::{
header::{CACHE_CONTROL, CONNECTION, CONTENT_TYPE},
StatusCode,
},
response::{IntoResponse, Response},
Json,
};
use std::collections::{HashMap, HashSet};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
api::admin::{
response::{
parse_editable_update, AdminEnvelope, AdminError, AdminResponse, BatchDeleteData,
EditableUpdateMessages, PageMeta, ADMIN_OK_CODE, ADMIN_OK_MESSAGE,
},
session::AdminAuth,
},
api::AppState,
fleet::{
account::AccountStatus,
manage::{
quota_view::{
quota_data, AccountQuotaData, AccountQuotaUsageWindow, AccountQuotaWindowLocalUsage,
},
AccountHealthCheck, AccountManageError, AccountRefreshOutcome, AccountRefreshResult,
AccountUpdate, ManagedAccount, OAuthExchangeInput,
},
refresh::token_refresh_status_eligible,
},
infra::{
format::{
format_cost, format_percent, format_plain_number, format_tokens, nonnegative_i64_to_u64,
},
json::{clamp_limit, clamp_page, total_pages},
time::{china_datetime, china_relative_time, china_rfc3339},
},
telemetry::{
account_usage::query::AccountUsageRecord,
billing,
buckets::query::{ModelUsageWindow, UsageBucketWindow},
},
};
const ACCOUNT_STATS_PAGE_LIMIT: u32 = 200;
const ACCOUNT_EXPORT_CONFIRMATION: &str = "export_sensitive_accounts";
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountsQuery {
page: Option<u32>,
page_size: Option<u32>,
search: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CreateAccountRequest {
token: Option<String>,
refresh_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct BatchDeleteAccountsRequest {
ids: Vec<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountActionRequest {
id: String,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountIdQuery {
id: String,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountExportQuery {
ids: Option<String>,
confirm: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountTestQuery {
id: String,
model_id: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AccountOAuthExchangeRequest {
session_id: String,
callback_url: Option<String>,
code: Option<String>,
state: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountModelsData {
models: Vec<AccountModelData>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountModelData {
id: String,
label: String,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountData {
id: String,
email: Option<String>,
account_id: Option<String>,
user_id: Option<String>,
label: Option<String>,
plan_type: Option<String>,
has_refresh_token: bool,
status: String,
display_status: String,
token_refreshing: bool,
access_token_expires_at: Option<String>,
access_token_expires_at_display: Option<String>,
added_at: String,
added_at_display: String,
updated_at: String,
updated_at_display: String,
quota: AccountQuotaData,
usage: AdminAccountUsageData,
}
impl From<ManagedAccount> for AdminAccountData {
fn from(a: ManagedAccount) -> Self {
Self::from_parts(a, None, None, Vec::new(), false)
}
}
impl AdminAccountData {
fn from_parts(
a: ManagedAccount,
usage: Option<&AccountUsageRecord>,
quota: Option<AccountQuotaData>,
models: Vec<AdminAccountModelUsageData>,
token_refreshing: bool,
) -> Self {
let access_token_expires_at = a.access_token_expires_at.as_ref().map(china_rfc3339);
let access_token_expires_at_display =
a.access_token_expires_at.as_ref().map(china_datetime);
let display_status = account_display_status(a.status, token_refreshing).to_string();
Self {
id: a.id,
email: a.email,
account_id: a.account_id,
user_id: a.user_id,
label: a.label,
plan_type: a.plan_type,
has_refresh_token: a.has_refresh_token,
status: a.status.as_str().to_string(),
display_status,
token_refreshing,
access_token_expires_at,
access_token_expires_at_display,
added_at: china_rfc3339(&a.added_at),
added_at_display: china_datetime(&a.added_at),
updated_at: china_rfc3339(&a.updated_at),
updated_at_display: china_datetime(&a.updated_at),
quota: quota.unwrap_or_default(),
usage: AdminAccountUsageData::from_usage(usage, models),
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountUsageData {
request_count: i64,
request_count_display: String,
empty_response_count: i64,
input_tokens: i64,
input_tokens_display: String,
output_tokens: i64,
output_tokens_display: String,
cached_tokens: i64,
cached_tokens_display: String,
reasoning_tokens: i64,
total_tokens: i64,
total_tokens_display: String,
image_input_tokens: i64,
image_output_tokens: i64,
image_tokens_display: String,
image_request_count: i64,
image_request_failed_count: i64,
created_tokens: i64,
created_tokens_display: String,
read_tokens: i64,
read_tokens_display: String,
last_used_at: Option<String>,
last_used_at_display: String,
models: Vec<AdminAccountModelUsageData>,
}
impl AdminAccountUsageData {
fn from_usage(
usage: Option<&AccountUsageRecord>,
models: Vec<AdminAccountModelUsageData>,
) -> Self {
let request_count = usage.map_or(0, |usage| usage.window_request_count);
let empty_response_count = 0;
let input_tokens = usage.map_or(0, |usage| usage.window_input_tokens);
let output_tokens = usage.map_or(0, |usage| usage.window_output_tokens);
let cached_tokens = usage.map_or(0, |usage| usage.window_cached_tokens);
let reasoning_tokens = 0;
let total_tokens = input_tokens.saturating_add(output_tokens);
let image_input_tokens = 0;
let image_output_tokens = 0;
let image_request_count = 0;
let image_request_failed_count = 0;
let created_tokens = input_tokens.saturating_sub(cached_tokens);
let read_tokens = cached_tokens;
let last_used_at = usage.and_then(|usage| usage.last_used_at);
Self {
request_count,
request_count_display: format_plain_number(nonnegative_i64_to_u64(request_count)),
empty_response_count,
input_tokens,
input_tokens_display: format_tokens(nonnegative_i64_to_u64(input_tokens)),
output_tokens,
output_tokens_display: format_tokens(nonnegative_i64_to_u64(output_tokens)),
cached_tokens,
cached_tokens_display: format_tokens(nonnegative_i64_to_u64(cached_tokens)),
reasoning_tokens,
total_tokens,
total_tokens_display: format_tokens(nonnegative_i64_to_u64(total_tokens)),
image_input_tokens,
image_output_tokens,
image_tokens_display: format_tokens(nonnegative_i64_to_u64(
image_input_tokens + image_output_tokens,
)),
image_request_count,
image_request_failed_count,
created_tokens,
created_tokens_display: format_tokens(nonnegative_i64_to_u64(created_tokens)),
read_tokens,
read_tokens_display: format_tokens(nonnegative_i64_to_u64(read_tokens)),
last_used_at: last_used_at.map(|value| china_rfc3339(&value)),
last_used_at_display: china_relative_time(last_used_at, Utc::now()),
models,
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountModelUsageData {
model: String,
request_count: u64,
request_count_display: String,
success_rate: f64,
success_rate_display: String,
input_tokens: u64,
input_tokens_display: String,
output_tokens: u64,
output_tokens_display: String,
cached_tokens: u64,
cached_tokens_display: String,
total_tokens: u64,
total_tokens_display: String,
total_cost_usd: f64,
total_cost_usd_display: String,
last_used_at: Option<String>,
last_used_at_display: String,
}
#[derive(Debug, Clone, Default)]
struct AccountListStats {
usage_by_account: HashMap<String, AccountUsageRecord>,
quota_by_account: HashMap<String, AccountQuotaData>,
models_by_account: HashMap<String, Vec<AdminAccountModelUsageData>>,
refreshing_account_ids: HashSet<String>,
}
struct AccountModelUsageRecord {
account_id: String,
model: String,
request_count: i64,
error_count: i64,
input_tokens: i64,
output_tokens: i64,
cached_tokens: i64,
total_cost_usd: f64,
last_used_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone)]
struct AccountQuotaWindowSelection {
account_id: String,
window: AccountQuotaUsageWindow,
}
impl AccountListStats {
fn data_for(&self, account: ManagedAccount) -> AdminAccountData {
let account_id = account.id.clone();
let token_refreshing = token_refresh_status_eligible(account.status)
&& self.refreshing_account_ids.contains(&account_id);
AdminAccountData::from_parts(
account,
self.usage_by_account.get(&account_id),
self.quota_by_account.get(&account_id).cloned(),
self.models_by_account
.get(&account_id)
.cloned()
.unwrap_or_default(),
token_refreshing,
)
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountSummaryData {
total: u64,
active: u64,
high_usage: u64,
attention: u64,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountPageData {
items: Vec<AdminAccountData>,
page: PageMeta,
summary: AdminAccountSummaryData,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountPageEnvelope {
code: u32,
message: String,
data: AdminAccountPageData,
}
impl AdminAccountPageEnvelope {
fn ok(
page: crate::infra::json::NumberedPage<AdminAccountData>,
summary: AdminAccountSummaryData,
) -> Self {
Self {
code: ADMIN_OK_CODE,
message: ADMIN_OK_MESSAGE.into(),
data: AdminAccountPageData {
page: PageMeta {
page: page.page,
page_size: page.page_size,
total: page.total,
total_pages: total_pages(page.total, page.page_size),
},
items: page.items,
summary,
},
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
enum AccountUpdateData {
Account(Box<AdminAccountData>),
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountRefreshData {
id: String,
email: Option<String>,
previous_status: String,
result: &'static str,
error: Option<String>,
duration_ms: i64,
}
impl From<AccountRefreshResult> for AccountRefreshData {
fn from(result: AccountRefreshResult) -> Self {
Self {
id: result.id,
email: result.email,
previous_status: result.previous_status.as_str().to_string(),
result: account_refresh_outcome_str(result.outcome),
error: result.error,
duration_ms: result.duration_ms,
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AdminAccountQuotaResponseData {
quota: Value,
raw: Value,
quota_data: AccountQuotaData,
plan_type: Option<String>,
account: AdminAccountData,
}
impl AdminAccountQuotaResponseData {
fn from_account(
quota: Value,
raw: Value,
quota_data: AccountQuotaData,
account: AdminAccountData,
) -> Self {
let plan_type = account.plan_type.clone();
Self {
quota,
raw,
quota_data,
plan_type,
account,
}
}
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(default)]
pub(crate) struct AccountHealthCheckRequest {
ids: Option<Vec<String>>,
stagger_ms: Option<u64>,
concurrency: Option<usize>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountHealthCheckSummaryData {
total: usize,
alive: usize,
dead: usize,
skipped: usize,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountHealthCheckData {
summary: AccountHealthCheckSummaryData,
results: Vec<AccountRefreshData>,
}
impl From<AccountHealthCheck> for AccountHealthCheckData {
fn from(result: AccountHealthCheck) -> Self {
let summary = AccountHealthCheckSummaryData {
total: result.results.len(),
alive: result.alive(),
dead: result.dead(),
skipped: result.skipped(),
};
Self {
summary,
results: result
.results
.into_iter()
.map(AccountRefreshData::from)
.collect(),
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountImportData {
imported: u32,
skipped: u32,
source_format: String,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountOAuthAuthorizeData {
session_id: String,
auth_url: String,
expires_at: String,
expires_at_display: String,
}
/// `GET /api/admin/accounts`
pub(crate) async fn accounts(
State(state): State<AppState>,
_auth: AdminAuth,
Query(params): Query<AccountsQuery>,
) -> Result<impl IntoResponse, AdminError> {
let page = clamp_page(params.page.unwrap_or(1));
let page_size = clamp_limit(params.page_size.unwrap_or(50));
let quota_by_account = quota_snapshots_by_account(&state).await;
let summary = account_summary_data(&state, &quota_by_account).await;
match state
.services
.admin_accounts
.list_page(page, page_size, params.search)
.await
{
Ok(page) => {
let stats = account_list_stats(&state, &page.items, &quota_by_account).await?;
let page = crate::infra::json::NumberedPage {
items: page
.items
.into_iter()
.map(|item| stats.data_for(item))
.collect(),
total: page.total,
page: page.page,
page_size: page.page_size,
};
Ok(AdminResponse::new(
StatusCode::OK,
AdminAccountPageEnvelope::ok(page, summary),
))
}
Err(error) => Err(account_error(&error)),
}
}
/// `POST /api/admin/accounts`
pub(crate) async fn create_account(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<CreateAccountRequest>,
) -> Result<impl IntoResponse, AdminError> {
match state
.services
.admin_accounts
.create(payload.token, payload.refresh_token)
.await
{
Ok(account) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AdminAccountData::from(account)),
)),
Err(error) => Err(account_error(&error)),
}
}
mod export_routes;
mod import_routes;
mod lifecycle_routes;
mod oauth_routes;
mod probe_routes;
mod query;
mod quota_routes;
pub(crate) use export_routes::*;
pub(crate) use import_routes::*;
pub(crate) use lifecycle_routes::*;
pub(crate) use oauth_routes::*;
pub(crate) use probe_routes::*;
use query::*;
pub(crate) use quota_routes::*;
@@ -0,0 +1,49 @@
use super::*;
/// `POST /api/admin/accounts/oauth/authorize`
pub async fn oauth_authorize_account(
State(state): State<AppState>,
_auth: AdminAuth,
) -> Result<impl IntoResponse, AdminError> {
match state.services.admin_accounts.oauth_authorize().await {
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountOAuthAuthorizeData {
session_id: result.session_id,
auth_url: result.auth_url,
expires_at: china_rfc3339(&result.expires_at),
expires_at_display: china_datetime(&result.expires_at),
}),
)),
Err(error) => Err(account_error(&error)),
}
}
/// `POST /api/admin/accounts/oauth/exchange`
pub async fn oauth_exchange_account(
State(state): State<AppState>,
_auth: AdminAuth,
Json(payload): Json<AccountOAuthExchangeRequest>,
) -> Result<impl IntoResponse, AdminError> {
match state
.services
.admin_accounts
.oauth_exchange(OAuthExchangeInput {
session_id: payload.session_id,
callback_url: payload.callback_url,
code: payload.code,
state: payload.state,
})
.await
{
Ok(result) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountImportData {
imported: result.imported,
skipped: result.skipped,
source_format: result.source_format.to_string(),
}),
)),
Err(error) => Err(account_error(&error)),
}
}
@@ -0,0 +1,62 @@
use super::*;
/// `GET /api/admin/accounts/test?id=...&modelId=...`
pub async fn test_account_connection(
State(state): State<AppState>,
Query(query): Query<AccountTestQuery>,
_auth: AdminAuth,
) -> Result<Response, AdminError> {
let account_id = query.id;
let model = query
.model_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
.ok_or_else(|| AdminError::bad_request("Model is required"))?;
let stream = state
.services
.admin_accounts
.test_connection_stream(&account_id, model)
.await
.map_err(|error| account_error(&error))?;
Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "text/event-stream")
.header(CACHE_CONTROL, "no-cache")
.header(CONNECTION, "keep-alive")
.header("x-accel-buffering", "no")
.body(Body::from_stream(stream))
.map_err(|_| AdminError::internal("Failed to build account test stream"))
}
/// `GET /api/admin/accounts/models?id=...`
pub async fn account_models(
State(state): State<AppState>,
Query(query): Query<AccountIdQuery>,
_auth: AdminAuth,
) -> Result<impl IntoResponse, AdminError> {
let account_id = query.id;
match state
.services
.admin_accounts
.account_models(&account_id)
.await
{
Ok(models) => Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AccountModelsData {
models: models
.into_iter()
.map(|model| AccountModelData {
id: model.id,
label: model.label,
})
.collect(),
}),
)),
Err(error) => Err(account_error(&error)),
}
}
@@ -0,0 +1,511 @@
use super::*;
pub(super) async fn account_list_stats(
state: &AppState,
accounts: &[ManagedAccount],
quota_by_account: &HashMap<String, AccountQuotaData>,
) -> Result<AccountListStats, AdminError> {
let account_ids = accounts
.iter()
.map(|account| account.id.clone())
.collect::<Vec<_>>();
let usage_records = state
.services
.usage
.list_by_account_ids(&account_ids)
.await
.unwrap_or_default();
let mut quota_by_account = quota_by_account.clone();
let all_quota_windows = quota_usage_windows_by_account(accounts, &quota_by_account);
let quota_window_usage_stats =
quota_window_local_usage_by_account(state, &all_quota_windows).await;
apply_quota_window_local_usage(accounts, &mut quota_by_account, &quota_window_usage_stats);
let selected_quota_windows = selected_quota_windows_by_account(accounts, &quota_by_account);
let usage_records = usage_records
.into_iter()
.map(|usage| {
apply_selected_quota_window_usage(
usage,
&selected_quota_windows,
&quota_window_usage_stats,
)
})
.collect::<Vec<_>>();
let models_by_account = list_current_window_model_usage(state, &usage_records).await;
let refreshing_account_ids = state
.services
.token_refresh
.refreshing_account_ids(&account_ids, Utc::now())
.await
.map_err(|error| AdminError::internal(error.to_string()))?;
Ok(AccountListStats {
usage_by_account: usage_records
.into_iter()
.map(|usage| (usage.account_id.clone(), usage))
.collect(),
quota_by_account,
models_by_account,
refreshing_account_ids,
})
}
pub(super) fn account_display_status(
status: AccountStatus,
token_refreshing: bool,
) -> &'static str {
if token_refreshing {
"refreshing"
} else {
status.as_str()
}
}
pub(super) fn selected_quota_windows_by_account(
accounts: &[ManagedAccount],
quota_by_account: &HashMap<String, AccountQuotaData>,
) -> HashMap<String, AccountQuotaUsageWindow> {
accounts
.iter()
.filter_map(|account| {
quota_by_account
.get(&account.id)
.and_then(selected_quota_window)
.map(|window| (account.id.clone(), window))
})
.collect()
}
pub(super) fn selected_quota_window(quota: &AccountQuotaData) -> Option<AccountQuotaUsageWindow> {
quota.usage_windows().into_iter().max_by(|left, right| {
left.duration_seconds()
.cmp(&right.duration_seconds())
.then_with(|| left.end.cmp(&right.end))
.then_with(|| left.key.cmp(&right.key))
})
}
pub(super) fn quota_usage_windows_by_account(
accounts: &[ManagedAccount],
quota_by_account: &HashMap<String, AccountQuotaData>,
) -> Vec<AccountQuotaWindowSelection> {
accounts
.iter()
.filter_map(|account| {
quota_by_account
.get(&account.id)
.map(|quota| (account.id.as_str(), quota))
})
.flat_map(|(account_id, quota)| {
quota
.usage_windows()
.into_iter()
.map(move |window| AccountQuotaWindowSelection {
account_id: account_id.to_string(),
window,
})
})
.collect()
}
pub(super) async fn apply_account_quota_window_local_usage(
state: &AppState,
account_id: &str,
quota: &mut AccountQuotaData,
) {
let windows = quota
.usage_windows()
.into_iter()
.map(|window| AccountQuotaWindowSelection {
account_id: account_id.to_string(),
window,
})
.collect::<Vec<_>>();
let usage_by_account = quota_window_local_usage_by_account(state, &windows).await;
let empty_usage = HashMap::new();
quota.apply_local_usage(usage_by_account.get(account_id).unwrap_or(&empty_usage));
}
pub(super) fn apply_quota_window_local_usage(
accounts: &[ManagedAccount],
quota_by_account: &mut HashMap<String, AccountQuotaData>,
usage_by_account: &HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>>,
) {
for account in accounts {
if let Some(quota) = quota_by_account.get_mut(&account.id) {
let empty_usage = HashMap::new();
quota.apply_local_usage(usage_by_account.get(&account.id).unwrap_or(&empty_usage));
}
}
}
pub(super) async fn quota_window_local_usage_by_account(
state: &AppState,
windows: &[AccountQuotaWindowSelection],
) -> HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>> {
let queries = windows
.iter()
.map(|window| UsageBucketWindow {
account_id: window.account_id.clone(),
key: window.window.key.clone(),
start: window.window.start,
end: window.window.end,
})
.collect::<Vec<_>>();
state
.services
.usage
.usage_by_windows(&queries)
.await
.unwrap_or_default()
.into_iter()
.map(|(account_id, usage_by_window)| {
let usage_by_window = usage_by_window
.into_iter()
.map(|(key, usage)| {
(
key,
AccountQuotaWindowLocalUsage {
request_count: usage.request_count,
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
cached_tokens: usage.cached_tokens,
},
)
})
.collect();
(account_id, usage_by_window)
})
.collect()
}
pub(super) fn apply_selected_quota_window_usage(
mut usage: AccountUsageRecord,
selected_quota_windows: &HashMap<String, AccountQuotaUsageWindow>,
quota_window_usage_stats: &HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>>,
) -> AccountUsageRecord {
let Some(window) = selected_quota_windows.get(&usage.account_id) else {
return usage;
};
let stats = quota_window_usage_stats
.get(&usage.account_id)
.and_then(|usage_by_window| usage_by_window.get(&window.key))
.copied()
.unwrap_or_default();
usage.window_request_count = stats.request_count;
usage.window_input_tokens = stats.input_tokens;
usage.window_output_tokens = stats.output_tokens;
usage.window_cached_tokens = stats.cached_tokens;
usage.window_started_at = Some(window.start);
usage.window_reset_at = Some(window.end);
usage.limit_window_seconds = Some(window.window_seconds);
usage
}
pub(super) async fn account_summary_data(
state: &AppState,
quota_by_account: &HashMap<String, AccountQuotaData>,
) -> AdminAccountSummaryData {
let accounts = list_all_account_metadata(state).await;
let total = accounts.len() as u64;
let active = accounts
.iter()
.filter(|account| account.status == AccountStatus::Active)
.count() as u64;
let high_usage = accounts
.iter()
.filter(|account| {
quota_by_account
.get(&account.id)
.is_some_and(account_quota_has_high_usage)
})
.count() as u64;
let attention = accounts
.iter()
.filter(|account| account_summary_needs_attention(account.status))
.count() as u64;
AdminAccountSummaryData {
total,
active,
high_usage,
attention,
}
}
pub(super) async fn quota_snapshots_by_account(
state: &AppState,
) -> HashMap<String, AccountQuotaData> {
state
.services
.admin_accounts
.quota_snapshots()
.await
.unwrap_or_default()
.into_iter()
.map(|snapshot| {
(
snapshot.account_id,
quota_data(&snapshot.quota_json, snapshot.quota_fetched_at),
)
})
.collect()
}
pub(super) async fn list_all_account_metadata(state: &AppState) -> Vec<ManagedAccount> {
let mut page = 1;
let mut accounts = Vec::new();
loop {
let Ok(result) = state
.services
.admin_accounts
.list_page(page, ACCOUNT_STATS_PAGE_LIMIT, None)
.await
else {
return Vec::new();
};
let total = result.total;
accounts.extend(result.items);
if accounts.len() as u64 >= total || total == 0 {
return accounts;
}
page = page.saturating_add(1);
}
}
pub(super) fn account_quota_has_high_usage(quota: &AccountQuotaData) -> bool {
quota.has_high_usage()
}
pub(super) fn account_summary_needs_attention(status: AccountStatus) -> bool {
matches!(
status,
AccountStatus::Expired | AccountStatus::Disabled | AccountStatus::Banned
)
}
pub(super) async fn list_current_window_model_usage(
state: &AppState,
usage_records: &[AccountUsageRecord],
) -> HashMap<String, Vec<AdminAccountModelUsageData>> {
let now = Utc::now();
let windows = usage_records
.iter()
.filter_map(|usage| {
current_usage_window(usage, now)
.map(|(start, end)| (usage.account_id.clone(), start, end))
})
.collect::<Vec<_>>();
if windows.is_empty() {
return HashMap::new();
}
let windows = windows
.into_iter()
.map(|(account_id, start, end)| ModelUsageWindow {
account_id,
start,
end,
})
.collect::<Vec<_>>();
let rows = state
.services
.usage
.model_usage_by_windows(&windows)
.await
.unwrap_or_default();
let mut records_by_model = HashMap::<(String, String), AccountModelUsageRecord>::new();
for row in rows {
let account_id = row.account_id;
let model = row.model;
let request_count = row.request_count;
let error_count = row.error_count;
let input_tokens = row.input_tokens;
let output_tokens = row.output_tokens;
let cached_tokens = row.cached_tokens;
let total_cost_usd = billing::calculate_cost(
nonnegative_i64_to_u64(input_tokens),
nonnegative_i64_to_u64(output_tokens),
nonnegative_i64_to_u64(cached_tokens),
&model,
row.service_tier.as_deref(),
);
let last_used_at = row.last_used_at;
let record = records_by_model
.entry((account_id.clone(), model.clone()))
.or_insert_with(|| AccountModelUsageRecord {
account_id,
model,
request_count: 0,
error_count: 0,
input_tokens: 0,
output_tokens: 0,
cached_tokens: 0,
total_cost_usd: 0.0,
last_used_at: None,
});
record.request_count += request_count;
record.error_count += error_count;
record.input_tokens += input_tokens;
record.output_tokens += output_tokens;
record.cached_tokens += cached_tokens;
record.total_cost_usd += total_cost_usd;
record.last_used_at = record.last_used_at.max(last_used_at);
}
let records = records_by_model.into_values().collect::<Vec<_>>();
models_by_account(records)
}
pub(super) fn current_usage_window(
usage: &AccountUsageRecord,
now: DateTime<Utc>,
) -> Option<(DateTime<Utc>, DateTime<Utc>)> {
let start = usage.window_started_at?;
let end = usage.window_reset_at.unwrap_or(now);
(start <= end).then_some((start, end))
}
pub(super) fn models_by_account(
records: Vec<AccountModelUsageRecord>,
) -> HashMap<String, Vec<AdminAccountModelUsageData>> {
let mut by_account = HashMap::<String, Vec<AccountModelUsageRecord>>::new();
for record in records {
by_account
.entry(record.account_id.clone())
.or_default()
.push(record);
}
by_account
.into_iter()
.map(|(account_id, mut records)| {
records.sort_by(|a, b| {
b.request_count
.cmp(&a.request_count)
.then_with(|| b.last_used_at.cmp(&a.last_used_at))
.then_with(|| a.model.cmp(&b.model))
});
(
account_id,
records.into_iter().map(model_usage_data).collect(),
)
})
.collect()
}
pub(super) fn model_usage_data(usage: AccountModelUsageRecord) -> AdminAccountModelUsageData {
let request_count = nonnegative_i64_to_u64(usage.request_count);
let error_count = nonnegative_i64_to_u64(usage.error_count);
let input_tokens = nonnegative_i64_to_u64(usage.input_tokens);
let output_tokens = nonnegative_i64_to_u64(usage.output_tokens);
let cached_tokens = nonnegative_i64_to_u64(usage.cached_tokens);
let total_tokens = input_tokens + output_tokens;
let success_rate = if request_count > 0 {
((request_count.saturating_sub(error_count)) as f64 / request_count as f64 * 1000.0).round()
/ 10.0
} else {
0.0
};
let total_cost_usd = usage.total_cost_usd;
AdminAccountModelUsageData {
model: usage.model,
request_count,
request_count_display: format_plain_number(request_count),
success_rate,
success_rate_display: format_percent(success_rate),
input_tokens,
input_tokens_display: format_tokens(input_tokens),
output_tokens,
output_tokens_display: format_tokens(output_tokens),
cached_tokens,
cached_tokens_display: format_tokens(cached_tokens),
total_tokens,
total_tokens_display: format_tokens(total_tokens),
total_cost_usd,
total_cost_usd_display: format_cost(total_cost_usd),
last_used_at: usage.last_used_at.map(|value| china_rfc3339(&value)),
last_used_at_display: china_relative_time(usage.last_used_at, Utc::now()),
}
}
pub(super) struct ParsedAccountUpdate {
pub(super) id: String,
pub(super) update: AccountUpdate,
}
pub(super) fn parse_account_update(payload: &Value) -> Result<ParsedAccountUpdate, AdminError> {
let payload = parse_editable_update(
payload,
EditableUpdateMessages {
object_required: "Account update request must be an object",
invalid: "Invalid account update request",
empty_update: "Account update request must include editable fields",
unknown_field_editable: true,
},
)?;
let update = AccountUpdate {
label: payload.label.map(|label| {
label.and_then(|value| {
let value = value.trim();
(!value.is_empty()).then(|| value.to_string())
})
}),
status: payload.status,
};
if !update.any() {
return Err(AdminError::bad_request(
"Account update request must include editable fields",
));
}
Ok(ParsedAccountUpdate {
id: payload.id,
update,
})
}
pub(super) fn account_error(error: &AccountManageError) -> AdminError {
match error {
AccountManageError::InvalidStatus(_)
| AccountManageError::LabelTooLong
| AccountManageError::EmptyIds
| AccountManageError::NoImportableAccounts
| AccountManageError::NoModels
| AccountManageError::InvalidAccessTokenExpiresAt
| AccountManageError::TokenRequired
| AccountManageError::InvalidToken(_)
| AccountManageError::RefreshTokenExchange(_)
| AccountManageError::OAuthSessionInvalid
| AccountManageError::OAuthCallbackInvalid
| AccountManageError::OAuthStateMismatch
| AccountManageError::NoValidCookies => AdminError::bad_request(error.to_string()),
AccountManageError::OAuthCodeExchange(_) => AdminError::bad_gateway(error.to_string()),
AccountManageError::NotFound => account_not_found(),
AccountManageError::Inactive(_) => AdminError::conflict(error.to_string()),
_ => AdminError::internal(error.to_string()),
}
}
pub(super) fn account_refresh_outcome_str(outcome: AccountRefreshOutcome) -> &'static str {
match outcome {
AccountRefreshOutcome::Alive => "alive",
AccountRefreshOutcome::Dead => "dead",
AccountRefreshOutcome::Skipped => "skipped",
}
}
pub(super) fn account_not_found() -> AdminError {
AdminError::not_found("Account not found")
}
pub(super) fn account_export_ids(value: Option<&str>) -> Vec<String> {
value
.into_iter()
.flat_map(|ids| ids.split(','))
.map(str::trim)
.filter(|id| !id.is_empty())
.map(ToString::to_string)
.collect()
}
@@ -0,0 +1,61 @@
use super::*;
/// `GET /api/admin/accounts/quota`
pub async fn account_quota(
State(state): State<AppState>,
_auth: AdminAuth,
Query(query): Query<AccountIdQuery>,
) -> Result<impl IntoResponse, AdminError> {
let account_id = query.id;
match state
.services
.admin_accounts
.account_quota(&account_id)
.await
{
Ok(data) => {
let quota = data.get("quota").cloned().unwrap_or(Value::Null);
let raw = data.get("raw").cloned().unwrap_or(Value::Null);
let quota_json = quota.to_string();
let mut quota_data = quota_data(&quota_json, Some(Utc::now()));
apply_account_quota_window_local_usage(&state, &account_id, &mut quota_data).await;
let account =
account_data_for_quota_refresh(&state, &account_id, quota_data.clone()).await?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AdminAccountQuotaResponseData::from_account(
quota, raw, quota_data, account,
)),
))
}
Err(AccountManageError::NotFound) => Err(account_not_found()),
Err(AccountManageError::Inactive(status)) => Err(AdminError::conflict(format!(
"Account is {}, cannot query quota",
status.as_str()
))),
Err(AccountManageError::FetchQuota(msg)) => Err(AdminError::bad_gateway(format!(
"Failed to fetch quota from Codex API: {msg}"
))),
Err(e) => Err(account_error(&e)),
}
}
async fn account_data_for_quota_refresh(
state: &AppState,
account_id: &str,
quota_data: AccountQuotaData,
) -> Result<AdminAccountData, AdminError> {
let Some(account) = state
.services
.admin_accounts
.get(account_id)
.await
.map_err(|error| account_error(&error))?
else {
return Err(account_not_found());
};
let quota_by_account = HashMap::from([(account_id.to_string(), quota_data)]);
let stats =
account_list_stats(state, std::slice::from_ref(&account), &quota_by_account).await?;
Ok(stats.data_for(account))
}
@@ -1,41 +1,28 @@
//! 管理端会话处理器。
//! 管理端认证路由。
use axum::{
extract::{FromRequestParts, State},
http::{header::SET_COOKIE, request::Parts, HeaderMap, HeaderValue, StatusCode},
extract::State,
http::{header::SET_COOKIE, HeaderMap, HeaderValue, StatusCode},
response::{IntoResponse, Response},
Extension, Json,
};
use serde::{Deserialize, Serialize};
use crate::{
admin::{
auth::service::AdminSessionError,
response::{AdminEnvelope, AdminError, AdminResponse},
api::{
admin::{
response::{AdminEnvelope, AdminError, AdminResponse},
session::{admin_session_cookie, ADMIN_SESSION_COOKIE},
},
middleware::request_id::ClientIp,
AppState,
},
http::middleware::request_id::ClientIp,
auth::types::SessionError,
infra::time::china_rfc3339,
runtime::state::AppState,
};
const ADMIN_SESSION_COOKIE: &str = "cpr_admin_session";
const ADMIN_SESSION_COOKIE_ATTRS: &str = "Path=/; Secure; HttpOnly; SameSite=Lax";
/// 已通过管理员会话或管理员 API Key 鉴权的请求。
pub(crate) struct AdminAuth;
impl FromRequestParts<AppState> for AdminAuth {
type Rejection = AdminError;
async fn from_request_parts(
parts: &mut Parts,
state: &AppState,
) -> Result<Self, Self::Rejection> {
require_admin_auth(state, &parts.headers).await?;
Ok(Self)
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AdminLoginRequest {
@@ -74,7 +61,7 @@ pub(crate) async fn login(
let session = match login_result {
Ok(Some(session)) => session,
Ok(None) => return Err(AdminError::invalid_admin_credentials()),
Err(AdminSessionError::LoginThrottled) => return Err(AdminError::too_many_login_attempts()),
Err(SessionError::LoginThrottled) => return Err(AdminError::too_many_login_attempts()),
Err(_) => return Err(AdminError::internal("Failed to create admin session")),
};
@@ -121,11 +108,14 @@ pub(crate) async fn logout(
headers: HeaderMap,
) -> Result<Response, AdminError> {
if let Some(session_id) = admin_session_cookie(&headers) {
let _ = state
if let Err(error) = state
.services
.admin_sessions
.delete_session(&session_id)
.await;
.await
{
tracing::warn!(error = %error, "failed to revoke admin session during logout");
}
}
let mut response = AdminResponse::new(
@@ -146,52 +136,6 @@ pub(crate) async fn logout(
Ok(response)
}
/// 要求请求携带有效管理员会话。
pub(crate) async fn require_admin_session(
state: &AppState,
headers: &HeaderMap,
) -> Result<(), AdminError> {
match state
.services
.admin_sessions
.validate(admin_session_cookie(headers).as_deref())
.await
{
Ok(true) => Ok(()),
Ok(false) => Err(AdminError::admin_session_required()),
Err(_) => Err(AdminError::internal("Failed to validate admin session")),
}
}
/// 要求请求携带有效管理员凭证:会话 Cookie 或管理员 API Key。
pub(crate) async fn require_admin_auth(
state: &AppState,
headers: &HeaderMap,
) -> Result<(), AdminError> {
if let Some(api_key) = admin_api_key_header(headers) {
return match state.services.settings.verify_admin_api_key(&api_key).await {
Ok(true) => Ok(()),
Ok(false) => Err(AdminError::invalid_admin_api_key()),
Err(_) => Err(AdminError::internal("Failed to validate admin API key")),
};
}
require_admin_session(state, headers).await
}
fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
let cookie = headers.get("cookie")?.to_str().ok()?;
cookie.split(';').find_map(|part| {
let (name, value) = part.trim().split_once('=')?;
(name == ADMIN_SESSION_COOKIE).then(|| value.to_string())
})
}
fn admin_api_key_header(headers: &HeaderMap) -> Option<String> {
let value = headers.get("x-api-key")?.to_str().ok()?.trim();
(!value.is_empty()).then(|| value.to_string())
}
fn optional_client_ip(client_ip: Option<Extension<ClientIp>>) -> Option<String> {
let Extension(client_ip) = client_ip?;
Some(client_ip.as_str().to_string())
+603
View File
@@ -0,0 +1,603 @@
//! 管理端 Dashboard 聚合视图。
use std::collections::{HashMap, HashSet};
use axum::{
extract::{Query, State},
http::StatusCode,
response::IntoResponse,
};
use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
api::admin::{
response::{AdminEnvelope, AdminError, AdminResponse},
session::AdminAuth,
usage_routes::{
usage_cost_details, usage_record_models, usage_token_details,
UsageRecordCostDetailsData, UsageRecordTokenDetailsData,
},
},
api::AppState,
fleet::{
account::{Account, AccountStatus},
pool::AccountCapacitySummary,
refresh::token_refresh_status_eligible,
},
infra::{
format::{format_duration_ms, format_tokens},
time::{
china_datetime, china_datetime_rfc3339_str, china_day_start, china_quarter_hour_start,
china_relative_time,
},
},
telemetry::{
account_usage::query::AccountUsageTimeBucket,
dashboard::{
dashboard_cards, dashboard_health_timeline_data, dashboard_trend_data,
DashboardAccountCounts, DashboardCardsData, DashboardHealthTimelineData,
DashboardTrendData, DashboardTrendKind,
},
usage::{
query::{UsageQueryFilter, UsageRecordAccountUsage},
types::{metadata_string, UsageRecord},
},
},
upstream::openai::fingerprint::Fingerprint,
};
const DASHBOARD_ACCOUNT_USAGE_LIMIT: u32 = 4;
const DASHBOARD_USAGE_RECORD_LIMIT: u32 = 10;
const DASHBOARD_TIME_BUCKET_MINUTES: i64 = 15;
const DASHBOARD_TIME_BUCKET_SLOTS: i64 = 7 * 24 * 4;
const FIVE_HOUR_WINDOW_SECONDS: u64 = 18_000;
const WEEK_WINDOW_SECONDS: u64 = 604_800;
const MONTH_WINDOW_SECONDS: u64 = 2_592_000;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct DashboardTrendQuery {
kind: Option<DashboardTrendKind>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardSummaryData {
cards: DashboardCardsData,
trend: DashboardTrendData,
health_timeline: DashboardHealthTimelineData,
account_usage: Vec<DashboardAccountUsageData>,
service_statuses: Vec<DashboardServiceStatusData>,
usage_records: Vec<DashboardUsageRecordData>,
pool_summary: AccountPoolDiagnostics,
capacity_info: AccountCapacityDiagnostics,
rotation_strategy: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardAccountUsageData {
id: String,
email: String,
plan_type: Option<String>,
tokens: String,
quota_used_percent: Option<f64>,
last_used: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardUsageRecordData {
id: String,
route: Option<String>,
model: String,
status_code: i64,
transport: Option<String>,
created_at_display: String,
account_email: Option<String>,
requested_model: Option<String>,
upstream_model: Option<String>,
client_ip: Option<String>,
user_agent: Option<String>,
reasoning_effort: Option<String>,
token_details: UsageRecordTokenDetailsData,
cost_details: Option<UsageRecordCostDetailsData>,
first_token_latency_ms_display: String,
latency_ms_display: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct DashboardServiceStatusData {
label: String,
value: String,
detail: String,
tone: String,
}
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountPoolDiagnostics {
total: usize,
active: usize,
expired: usize,
quota_exhausted: usize,
refreshing: usize,
disabled: usize,
banned: usize,
}
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountCapacityDiagnostics {
max_concurrent_per_account: usize,
total_slots: usize,
used_slots: usize,
available_slots: usize,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct FingerprintDiagnostics {
source: &'static str,
originator: String,
app_version: String,
build_number: String,
platform: String,
arch: String,
chromium_version: String,
user_agent: String,
#[serde(skip_serializing_if = "Option::is_none")]
updated_at: Option<String>,
}
fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics {
FingerprintDiagnostics {
source: "runtime",
originator: fingerprint.originator.clone(),
app_version: fingerprint.app_version.clone(),
build_number: fingerprint.build_number.clone(),
platform: fingerprint.platform.clone(),
arch: fingerprint.arch.clone(),
chromium_version: fingerprint.chromium_version.clone(),
user_agent: fingerprint.user_agent(),
updated_at: fingerprint.updated_at.clone(),
}
}
impl From<AccountCapacitySummary> for AccountCapacityDiagnostics {
fn from(summary: AccountCapacitySummary) -> Self {
Self {
max_concurrent_per_account: summary.max_concurrent_per_account,
total_slots: summary.total_slots,
used_slots: summary.used_slots,
available_slots: summary.available_slots,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum QuotaWindowPriority {
FiveHour,
Weekly,
Monthly,
Other,
}
/// `GET /api/admin/dashboard/summary`
pub(crate) async fn dashboard_summary(
State(state): State<AppState>,
_auth: AdminAuth,
Query(query): Query<DashboardTrendQuery>,
) -> Result<impl IntoResponse, AdminError> {
let accounts = state
.services
.accounts
.list_pool_accounts()
.await
.map_err(|error| dashboard_data_error("accounts", &error))?;
let capacity = state.services.account_pool.capacity_summary_now().await;
let now = Utc::now();
let today_filter = today_usage_record_filter(now);
let lifetime_usage = state
.services
.usage
.summary()
.await
.map_err(|error| dashboard_data_error("account usage summary", &error))?;
let account_usage_records = state
.services
.usage_records
.account_usage(today_filter.clone(), DASHBOARD_ACCOUNT_USAGE_LIMIT)
.await
.map_err(|error| dashboard_data_error("account usage ranking", &error))?;
let recent_events = state
.services
.usage_records
.list_recent(DASHBOARD_USAGE_RECORD_LIMIT, today_filter)
.await
.map_err(|error| dashboard_data_error("recent usage records", &error))?;
let time_buckets = dashboard_time_buckets(&state, now).await?;
let account_ids = accounts
.iter()
.map(|account| account.id.clone())
.collect::<Vec<_>>();
let refreshing_account_ids = state
.services
.token_refresh
.refreshing_account_ids(&account_ids, now)
.await
.map_err(|error| dashboard_data_error("refreshing accounts", &error))?;
let pool_summary = account_pool_summary(&accounts, &refreshing_account_ids);
let trend = dashboard_trend_data(&time_buckets, query.kind.unwrap_or_default());
let quota_used_by_account =
account_quota_used_percent_by_id(&state, &account_usage_records).await?;
let settings = state.services.settings.current();
let recent_usage_records = recent_events;
let account_emails = state
.services
.usage_records
.account_email_map(&recent_usage_records)
.await
.map_err(|error| AdminError::usage_record_accounts_failed(error.to_string()))?;
let dashboard_usage_records =
dashboard_usage_record_items(recent_usage_records, &account_emails);
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(DashboardSummaryData {
cards: dashboard_cards(
dashboard_account_counts(&accounts),
&time_buckets,
&lifetime_usage,
),
trend,
health_timeline: dashboard_health_timeline_data(&time_buckets),
account_usage: account_usage_data(
&accounts,
&account_usage_records,
&quota_used_by_account,
now,
),
service_statuses: service_status_data(&state),
usage_records: dashboard_usage_records,
pool_summary,
capacity_info: AccountCapacityDiagnostics::from(capacity),
rotation_strategy: Some(settings.rotation_strategy.clone()),
}),
))
}
fn today_usage_record_filter(now: DateTime<Utc>) -> UsageQueryFilter {
UsageQueryFilter {
start_time: Some(china_day_start(now)),
end_time: Some(now),
..UsageQueryFilter::default()
}
}
/// `GET /api/admin/dashboard/trend?kind=usage|latency|errors`
pub(crate) async fn dashboard_trend(
State(state): State<AppState>,
_auth: AdminAuth,
Query(query): Query<DashboardTrendQuery>,
) -> Result<impl IntoResponse, AdminError> {
let time_buckets = dashboard_time_buckets(&state, Utc::now()).await?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(dashboard_trend_data(
&time_buckets,
query.kind.unwrap_or_default(),
)),
))
}
async fn dashboard_time_buckets(
state: &AppState,
now: DateTime<Utc>,
) -> Result<Vec<AccountUsageTimeBucket>, AdminError> {
let current_slot = china_quarter_hour_start(now);
let start = current_slot
- Duration::minutes(DASHBOARD_TIME_BUCKET_MINUTES * (DASHBOARD_TIME_BUCKET_SLOTS - 1));
state
.services
.usage
.time_buckets(start, now)
.await
.map_err(|error| dashboard_data_error("time buckets", &error))
}
fn dashboard_account_counts(accounts: &[Account]) -> DashboardAccountCounts {
DashboardAccountCounts {
total: accounts.len() as u64,
enabled: accounts
.iter()
.filter(|account| account.status == AccountStatus::Active)
.count() as u64,
abnormal: accounts
.iter()
.filter(|account| account.status != AccountStatus::Active)
.count() as u64,
}
}
fn account_pool_summary(
accounts: &[Account],
refreshing_account_ids: &HashSet<String>,
) -> AccountPoolDiagnostics {
let mut summary = AccountPoolDiagnostics {
total: accounts.len(),
..AccountPoolDiagnostics::default()
};
for account in accounts {
match account.status {
status
if token_refresh_status_eligible(status)
&& refreshing_account_ids.contains(&account.id) =>
{
summary.refreshing += 1;
}
AccountStatus::Active => summary.active += 1,
AccountStatus::Expired => summary.expired += 1,
AccountStatus::QuotaExhausted => summary.quota_exhausted += 1,
AccountStatus::Disabled => summary.disabled += 1,
AccountStatus::Banned => summary.banned += 1,
}
}
summary
}
fn account_usage_data(
accounts: &[Account],
usage_records: &[UsageRecordAccountUsage],
quota_used_by_account: &HashMap<String, f64>,
now: DateTime<Utc>,
) -> Vec<DashboardAccountUsageData> {
let account_by_id = accounts
.iter()
.map(|account| (account.id.as_str(), account))
.collect::<HashMap<_, _>>();
usage_records
.iter()
.map(|usage| {
let account = account_by_id.get(usage.account_id.as_str()).copied();
let quota_used_percent = quota_used_by_account
.get(&usage.account_id)
.copied()
.or_else(|| {
matches!(
account.map(|account| account.status),
Some(AccountStatus::QuotaExhausted)
)
.then_some(100.0)
});
DashboardAccountUsageData {
id: usage.account_id.clone(),
email: account
.and_then(|account| account.email.as_ref())
.cloned()
.unwrap_or_else(|| usage.account_id.clone()),
plan_type: account
.and_then(|account| account.plan_type.as_ref())
.cloned(),
tokens: format_tokens(usage.total_tokens),
quota_used_percent,
last_used: china_relative_time(Some(usage.last_used_at), now),
}
})
.collect()
}
async fn account_quota_used_percent_by_id(
state: &AppState,
usage_records: &[UsageRecordAccountUsage],
) -> Result<HashMap<String, f64>, AdminError> {
let mut quota_used_by_account = HashMap::with_capacity(usage_records.len());
for usage in usage_records {
let quota_json = state
.services
.accounts
.get_quota_json(&usage.account_id)
.await
.map_err(|error| dashboard_data_error("account quota", &error))?;
let Some(quota_json) = quota_json else {
continue;
};
if let Some(used_percent) = quota_used_percent(&quota_json) {
quota_used_by_account.insert(usage.account_id.clone(), used_percent);
}
}
Ok(quota_used_by_account)
}
fn dashboard_data_error(source: &'static str, error: &impl std::fmt::Display) -> AdminError {
tracing::error!(source, error = %error, "failed to load dashboard data");
AdminError::internal("Failed to load dashboard data")
}
fn quota_used_percent(quota_json: &str) -> Option<f64> {
let quota = serde_json::from_str::<Value>(quota_json).ok()?;
let mut selected = None;
if let Some(monthly_limit) = quota.get("monthly_limit") {
select_quota_used_percent(
&mut selected,
quota_window_priority(monthly_limit, QuotaWindowPriority::Monthly),
monthly_limit.get("used_percent").and_then(percent_value),
);
}
if let Some(snapshots) = quota.get("snapshots").and_then(Value::as_array) {
for snapshot in snapshots {
for role in ["primary", "secondary"] {
if let Some(window) = snapshot.get(role) {
select_quota_used_percent(
&mut selected,
quota_window_priority(window, QuotaWindowPriority::Other),
window.get("used_percent").and_then(percent_value),
);
}
}
}
}
selected.map(|(_, used_percent)| used_percent)
}
fn select_quota_used_percent(
selected: &mut Option<(QuotaWindowPriority, f64)>,
priority: QuotaWindowPriority,
used_percent: Option<f64>,
) {
let Some(used_percent) = used_percent else {
return;
};
match selected {
Some((selected_priority, selected_percent))
if priority > *selected_priority
|| (priority == *selected_priority && used_percent <= *selected_percent) => {}
_ => *selected = Some((priority, used_percent)),
}
}
fn quota_window_priority(window: &Value, fallback: QuotaWindowPriority) -> QuotaWindowPriority {
let window_seconds = window
.get("window_minutes")
.and_then(Value::as_u64)
.and_then(|minutes| minutes.checked_mul(60));
match window_seconds {
Some(seconds) if quota_window_matches(seconds, FIVE_HOUR_WINDOW_SECONDS) => {
QuotaWindowPriority::FiveHour
}
Some(seconds) if quota_window_matches(seconds, WEEK_WINDOW_SECONDS) => {
QuotaWindowPriority::Weekly
}
Some(seconds) if quota_window_matches(seconds, MONTH_WINDOW_SECONDS) => {
QuotaWindowPriority::Monthly
}
_ => fallback,
}
}
fn quota_window_matches(actual: u64, expected: u64) -> bool {
actual > 0 && actual.abs_diff(expected) <= expected / 20
}
fn percent_value(value: &Value) -> Option<f64> {
let percent = value
.as_f64()
.or_else(|| value.as_str().and_then(|value| value.parse::<f64>().ok()))?;
percent.is_finite().then_some(percent.clamp(0.0, 100.0))
}
fn dashboard_usage_record_items(
items: Vec<UsageRecord>,
account_emails: &HashMap<String, String>,
) -> Vec<DashboardUsageRecordData> {
items
.into_iter()
.map(|record| dashboard_usage_record_data(record, account_emails))
.collect()
}
fn dashboard_usage_record_data(
record: UsageRecord,
account_emails: &HashMap<String, String>,
) -> DashboardUsageRecordData {
let account_email = account_emails
.get(&record.account_id)
.cloned()
.or_else(|| metadata_string(&record.metadata, &["accountEmail", "account_email"]));
let (requested_model, upstream_model) = usage_record_models(&record);
let client_ip = metadata_string(&record.metadata, &["clientIp", "ipAddress", "ip_address"]);
let user_agent = metadata_string(&record.metadata, &["userAgent", "user_agent"]);
let reasoning_effort =
metadata_string(&record.metadata, &["reasoningEffort", "reasoning_effort"]);
let token_details = usage_token_details(&record);
let cost_details = usage_cost_details(&record, upstream_model.as_deref(), &token_details);
let first_token_latency_ms = record.first_token_ms;
DashboardUsageRecordData {
id: record.id,
route: record.route,
model: record.model,
status_code: record.status_code,
transport: record.transport,
created_at_display: china_datetime(&record.created_at),
account_email,
requested_model,
upstream_model,
client_ip,
user_agent,
reasoning_effort,
token_details,
cost_details,
first_token_latency_ms_display: format_duration_ms(first_token_latency_ms),
latency_ms_display: format_duration_ms(record.latency_ms),
}
}
fn service_status_data(state: &AppState) -> Vec<DashboardServiceStatusData> {
let current_fingerprint = state.services.fingerprint.snapshot();
let fingerprint = fingerprint_diagnostics(&current_fingerprint);
vec![
DashboardServiceStatusData {
label: "客户端版本".to_string(),
value: empty_dash(fingerprint.app_version),
detail: if fingerprint.build_number.trim().is_empty() {
"-".to_string()
} else {
format!("Build {}", fingerprint.build_number)
},
tone: "info".to_string(),
},
DashboardServiceStatusData {
label: "平台架构".to_string(),
value: empty_dash(fingerprint.platform),
detail: empty_dash(fingerprint.arch),
tone: "info".to_string(),
},
DashboardServiceStatusData {
label: "Chromium".to_string(),
value: if fingerprint.chromium_version.trim().is_empty() {
"-".to_string()
} else {
format!("v{}", fingerprint.chromium_version)
},
detail: empty_dash(fingerprint.originator),
tone: "normal".to_string(),
},
DashboardServiceStatusData {
label: "更新时间".to_string(),
value: format_fingerprint_updated_at(fingerprint.updated_at),
detail: String::new(),
tone: "normal".to_string(),
},
DashboardServiceStatusData {
label: "User Agent".to_string(),
value: fingerprint.user_agent,
detail: String::new(),
tone: "normal".to_string(),
},
]
}
fn empty_dash(value: String) -> String {
if value.trim().is_empty() {
"-".to_string()
} else {
value
}
}
fn format_fingerprint_updated_at(value: Option<String>) -> String {
let Some(value) = value else {
return "-".to_string();
};
let value = value.trim();
if value.is_empty() {
return "-".to_string();
}
china_datetime_rfc3339_str(value)
}
@@ -10,24 +10,25 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
admin::auth::session::AdminAuth,
admin::keys::service::{AdminClientApiKey, AdminClientKeyError},
admin::response::{
AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse, BatchDeleteData,
api::admin::response::{
parse_editable_update, AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse,
BatchDeleteData, EditableUpdateMessages,
},
admin::update_payload::{parse_editable_update, EditableUpdateMessages},
api::admin::session::AdminAuth,
api::AppState,
infra::{
json::{clamp_limit, Page},
json::{clamp_limit, clamp_page, NumberedPage},
time::{china_relative_time_str, china_rfc3339_str},
},
runtime::state::AppState,
keys::types::{KeyManageError, ManagedClientApiKey},
};
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ApiKeysQuery {
cursor: Option<String>,
limit: Option<u32>,
page: Option<u32>,
page_size: Option<u32>,
search: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -57,8 +58,8 @@ struct ClientApiKeyData {
last_used_at_display: String,
}
impl From<AdminClientApiKey> for ClientApiKeyData {
fn from(k: AdminClientApiKey) -> Self {
impl From<ManagedClientApiKey> for ClientApiKeyData {
fn from(k: ManagedClientApiKey) -> Self {
Self {
id: k.id,
name: k.name,
@@ -74,12 +75,12 @@ impl From<AdminClientApiKey> for ClientApiKeyData {
}
}
fn client_key_error(error: &AdminClientKeyError) -> AdminError {
fn client_key_error(error: &KeyManageError) -> AdminError {
match error {
AdminClientKeyError::InvalidStatus(_)
| AdminClientKeyError::EmptyName
| AdminClientKeyError::EmptyIds
| AdminClientKeyError::LabelTooLong => AdminError::bad_request(error.to_string()),
KeyManageError::InvalidStatus(_)
| KeyManageError::EmptyName
| KeyManageError::EmptyIds
| KeyManageError::LabelTooLong => AdminError::bad_request(error.to_string()),
_ => AdminError::internal(error.to_string()),
}
}
@@ -117,21 +118,24 @@ pub(crate) async fn api_keys(
_auth: AdminAuth,
Query(query): Query<ApiKeysQuery>,
) -> Result<impl IntoResponse, AdminError> {
let limit = clamp_limit(query.limit.unwrap_or(50));
let page = clamp_page(query.page.unwrap_or(1));
let page_size = clamp_limit(query.page_size.unwrap_or(50));
match state
.services
.admin_client_keys
.list(query.cursor, limit)
.list_page(page, page_size, query.search)
.await
{
Ok(page) => {
let page = Page {
let page = NumberedPage {
items: page.items.into_iter().map(ClientApiKeyData::from).collect(),
next_cursor: page.next_cursor,
total: page.total,
page: page.page,
page_size: page.page_size,
};
Ok(AdminResponse::new(
StatusCode::OK,
AdminPageEnvelope::ok(page, limit),
AdminPageEnvelope::ok(page),
))
}
Err(error) => Err(client_key_error(&error)),
+13
View File
@@ -0,0 +1,13 @@
//! 管理端 HTTP API。
pub mod accounts_routes;
pub mod auth_routes;
pub mod dashboard_routes;
pub mod keys_routes;
pub mod ops_routes;
pub mod response;
pub mod router;
pub mod session;
pub mod settings_routes;
pub mod system_routes;
pub mod usage_routes;
+135
View File
@@ -0,0 +1,135 @@
//! 运维错误明细 HTTP 处理器。
use axum::{
extract::{Query, State},
http::StatusCode,
response::IntoResponse,
};
use serde::{Deserialize, Serialize};
use crate::{
api::admin::{
response::{AdminError, AdminPageEnvelope, AdminResponse},
session::AdminAuth,
},
api::AppState,
infra::{
json::{clamp_limit, clamp_page, NumberedPage},
time::china_datetime,
},
telemetry::{ops::query::OpsQueryError, ops::store::OpsErrorFilter, ops::types::OpsErrorLog},
};
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct OpsErrorsQuery {
page: Option<u32>,
page_size: Option<u32>,
kind: Option<String>,
client_api_key_id: Option<String>,
provider: Option<String>,
request_id: Option<String>,
account_id: Option<String>,
route: Option<String>,
model: Option<String>,
status_code: Option<i64>,
client_status_code: Option<i64>,
upstream_status_code: Option<i64>,
transport: Option<String>,
attempt_index: Option<i64>,
failure_class: Option<String>,
response_id: Option<String>,
upstream_request_id: Option<String>,
search: Option<String>,
start_time: Option<String>,
end_time: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct OpsErrorData {
#[serde(flatten)]
error: OpsErrorLog,
created_at_display: String,
}
/// `GET /api/admin/ops/errors`
pub(crate) async fn ops_errors(
State(state): State<AppState>,
_auth: AdminAuth,
Query(query): Query<OpsErrorsQuery>,
) -> Result<impl IntoResponse, AdminError> {
let page = clamp_page(query.page.unwrap_or(1));
let page_size = clamp_limit(query.page_size.unwrap_or(50));
let filter = filter_from_query(query)?;
state
.services
.ops_errors
.list_page(page, page_size, filter)
.await
.map(|page| {
let page = NumberedPage {
items: ops_error_items(page.items),
total: page.total,
page: page.page,
page_size: page.page_size,
};
AdminResponse::new(StatusCode::OK, AdminPageEnvelope::ok(page))
})
.map_err(log_error)
}
fn filter_from_query(query: OpsErrorsQuery) -> Result<OpsErrorFilter, AdminError> {
Ok(OpsErrorFilter {
kind: non_empty(query.kind),
client_api_key_id: non_empty(query.client_api_key_id),
provider: non_empty(query.provider),
request_id: non_empty(query.request_id),
account_id: non_empty(query.account_id),
route: non_empty(query.route),
model: non_empty(query.model),
status_code: query.status_code,
client_status_code: query.client_status_code,
upstream_status_code: query.upstream_status_code,
transport: non_empty(query.transport),
attempt_index: query.attempt_index,
failure_class: non_empty(query.failure_class),
response_id: non_empty(query.response_id),
upstream_request_id: non_empty(query.upstream_request_id),
search: non_empty(query.search),
start_time: optional_datetime(query.start_time)?,
end_time: optional_datetime(query.end_time)?,
})
}
fn ops_error_items(items: Vec<OpsErrorLog>) -> Vec<OpsErrorData> {
items
.into_iter()
.map(|error| OpsErrorData {
created_at_display: china_datetime(&error.created_at),
error,
})
.collect()
}
fn optional_datetime(
value: Option<String>,
) -> Result<Option<chrono::DateTime<chrono::Utc>>, AdminError> {
let Some(value) = non_empty(value) else {
return Ok(None);
};
chrono::DateTime::parse_from_rfc3339(&value)
.map(|value| Some(value.with_timezone(&chrono::Utc)))
.map_err(|_| AdminError::invalid_time_range("Invalid time range"))
}
fn non_empty(value: Option<String>) -> Option<String> {
value
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn log_error(error: OpsQueryError) -> AdminError {
AdminError::internal(error.to_string())
}
@@ -1,12 +1,13 @@
//! 管理端响应封装与 From 转换。
use crate::infra::json::{total_pages, NumberedPage, Page};
use crate::infra::json::{total_pages, NumberedPage};
use axum::{
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::fmt;
pub(crate) const ADMIN_OK_CODE: u32 = 200;
@@ -186,33 +187,16 @@ impl<T> AdminEnvelope<T> {
}
}
/// 分页元数据。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CursorPageMeta {
pub(crate) limit: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) next_cursor: Option<String>,
}
/// 页码分页元数据。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct NumberedPageMeta {
pub(crate) struct PageMeta {
pub(crate) page: u32,
pub(crate) page_size: u32,
pub(crate) total: u64,
pub(crate) total_pages: u32,
}
/// 分页元数据。
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub(crate) enum PageMeta {
Cursor(CursorPageMeta),
Numbered(NumberedPageMeta),
}
/// 分页响应信封。
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
@@ -239,19 +223,7 @@ pub(crate) struct BatchDeleteData {
}
impl<T> AdminPageEnvelope<T> {
pub fn ok(page: Page<T>, limit: u32) -> Self {
let Page { items, next_cursor } = page;
Self {
code: ADMIN_OK_CODE,
message: ADMIN_OK_MESSAGE.into(),
data: PageData {
items,
page: PageMeta::Cursor(CursorPageMeta { limit, next_cursor }),
},
}
}
pub fn numbered(page: NumberedPage<T>) -> Self {
pub fn ok(page: NumberedPage<T>) -> Self {
let NumberedPage {
items,
total,
@@ -263,12 +235,12 @@ impl<T> AdminPageEnvelope<T> {
message: ADMIN_OK_MESSAGE.into(),
data: PageData {
items,
page: PageMeta::Numbered(NumberedPageMeta {
page: PageMeta {
page,
page_size,
total,
total_pages: total_pages(total, page_size),
}),
},
},
}
}
@@ -291,3 +263,59 @@ impl<T: Serialize> IntoResponse for AdminResponse<T> {
(self.status, Json(self.body)).into_response()
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct EditableUpdateRequest {
id: String,
label: Option<Option<String>>,
status: Option<String>,
}
pub(crate) struct ParsedEditableUpdate {
pub id: String,
pub label: Option<Option<String>>,
pub status: Option<String>,
}
pub(crate) struct EditableUpdateMessages<'a> {
pub object_required: &'a str,
pub invalid: &'a str,
pub empty_update: &'a str,
pub unknown_field_editable: bool,
}
pub(crate) fn parse_editable_update(
payload: &Value,
messages: EditableUpdateMessages<'_>,
) -> Result<ParsedEditableUpdate, AdminError> {
let object = payload
.as_object()
.ok_or_else(|| AdminError::bad_request(messages.object_required))?;
if messages.unknown_field_editable {
for field in object.keys() {
if !matches!(field.as_str(), "id" | "label" | "status") {
return Err(AdminError::bad_request(format!("{field} is not editable")));
}
}
}
let update = serde_json::from_value::<EditableUpdateRequest>(payload.clone())
.map_err(|_| AdminError::bad_request(messages.invalid))?;
if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) {
return Err(AdminError::bad_request(messages.invalid));
}
if update.label.is_none() && update.status.is_none() {
return Err(AdminError::bad_request(messages.empty_update));
}
Ok(ParsedEditableUpdate {
id: update.id,
label: update.label,
status: update.status,
})
}
fn is_blank(value: &str) -> bool {
value.trim().is_empty()
}
@@ -1,36 +1,38 @@
//! 管理端路由。
use axum::{
http::{header, HeaderValue},
middleware,
response::Response,
routing::{get, post},
Router,
};
use crate::runtime::state::AppState;
use crate::api::AppState;
use super::{
accounts::routes::{
accounts_routes::{
account_models, account_quota, accounts, batch_delete_accounts, create_account,
export_accounts, health_check_accounts, import_accounts, oauth_authorize_account,
oauth_exchange_account, refresh_account, test_account_connection, update_account,
},
auth::session::{login, logout, session_status},
keys::routes::{api_keys, batch_delete_api_keys, create_api_key, update_api_key},
monitoring::{
dashboard::{dashboard_summary, dashboard_trend},
usage_record_routes::{
clear_usage_records, usage_record_detail, usage_records,
usage_records_endpoint_distribution, usage_records_latency_trend,
usage_records_model_distribution, usage_records_summary, usage_records_token_trend,
},
},
settings::routes::{
auth_routes::{login, logout, session_status},
dashboard_routes::{dashboard_summary, dashboard_trend},
keys_routes::{api_keys, batch_delete_api_keys, create_api_key, update_api_key},
ops_routes::ops_errors,
settings_routes::{
admin_api_key_status, delete_admin_api_key, regenerate_admin_api_key, settings,
update_settings,
},
system::routes::{
system_routes::{
perform_update, restart, rollback, update_detail, update_event_stream, update_status,
version,
},
usage_routes::{
clear_usage_records, usage_record_detail, usage_records,
usage_records_endpoint_distribution, usage_records_latency_trend,
usage_records_model_distribution, usage_records_summary, usage_records_token_trend,
},
};
/// 构造管理端路由。
@@ -101,7 +103,16 @@ pub fn router() -> Router<AppState> {
)
.route("/api/admin/usage/records/delete", post(clear_usage_records))
.route("/api/admin/usage/records/detail", get(usage_record_detail))
.route("/api/admin/ops/errors", get(ops_errors))
.route("/api/admin/keys", get(api_keys).post(create_api_key))
.route("/api/admin/keys/delete", post(batch_delete_api_keys))
.route("/api/admin/keys/update", post(update_api_key))
.layer(middleware::map_response(no_store))
}
async fn no_store(mut response: Response) -> Response {
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
response
}
+71
View File
@@ -0,0 +1,71 @@
//! 管理端认证提取器与会话鉴权。
use axum::{
extract::FromRequestParts,
http::{request::Parts, HeaderMap},
};
use crate::api::{admin::response::AdminError, AppState};
pub(super) const ADMIN_SESSION_COOKIE: &str = "cpr_admin_session";
/// 已通过管理员会话或管理员 API Key 鉴权的请求。
pub(crate) struct AdminAuth;
impl FromRequestParts<AppState> for AdminAuth {
type Rejection = AdminError;
async fn from_request_parts(
parts: &mut Parts,
state: &AppState,
) -> Result<Self, Self::Rejection> {
require_admin_auth(state, &parts.headers).await?;
Ok(Self)
}
}
/// 要求请求携带有效管理员会话。
pub(crate) async fn require_admin_session(
state: &AppState,
headers: &HeaderMap,
) -> Result<(), AdminError> {
match state
.services
.admin_sessions
.validate(admin_session_cookie(headers).as_deref())
.await
{
Ok(true) => Ok(()),
Ok(false) => Err(AdminError::admin_session_required()),
Err(_) => Err(AdminError::internal("Failed to validate admin session")),
}
}
/// 要求请求携带有效管理员凭证:会话 Cookie 或管理员 API Key。
pub(crate) async fn require_admin_auth(
state: &AppState,
headers: &HeaderMap,
) -> Result<(), AdminError> {
if let Some(api_key) = admin_api_key_header(headers) {
return match state.services.settings.verify_admin_api_key(&api_key).await {
Ok(true) => Ok(()),
Ok(false) => Err(AdminError::invalid_admin_api_key()),
Err(_) => Err(AdminError::internal("Failed to validate admin API key")),
};
}
require_admin_session(state, headers).await
}
pub(super) fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
let cookie = headers.get("cookie")?.to_str().ok()?;
cookie.split(';').find_map(|part| {
let (name, value) = part.trim().split_once('=')?;
(name == ADMIN_SESSION_COOKIE).then(|| value.to_string())
})
}
fn admin_api_key_header(headers: &HeaderMap) -> Option<String> {
let value = headers.get("x-api-key")?.to_str().ok()?.trim();
(!value.is_empty()).then(|| value.to_string())
}
@@ -6,19 +6,16 @@ use serde_json::Value;
use std::collections::BTreeMap;
use crate::{
admin::auth::session::AdminAuth,
admin::response::{AdminEnvelope, AdminError, AdminResponse},
config::{
schema::AppConfig,
settings::{AdminSettingsPatch, RuntimeSettingsError, SettingsServiceError},
api::admin::{
response::{AdminEnvelope, AdminError, AdminResponse},
session::AdminAuth,
},
runtime::state::AppState,
upstream::models::config::ModelConfig,
api::AppState,
settings::{SettingsError, SettingsPatch, SettingsSnapshot, SettingsValidationError},
};
const ALLOWED_SETTINGS_KEYS: [&str; 7] = [
const ALLOWED_SETTINGS_KEYS: [&str; 6] = [
"modelAliases",
"modelAccountRoutes",
"refreshMarginSeconds",
"refreshConcurrency",
"maxConcurrentPerAccount",
@@ -30,7 +27,6 @@ const ALLOWED_SETTINGS_KEYS: [&str; 7] = [
#[serde(rename_all = "camelCase")]
struct AdminSettingsData {
model_aliases: BTreeMap<String, String>,
model_account_routes: BTreeMap<String, Vec<String>>,
refresh_margin_seconds: u64,
refresh_concurrency: u32,
max_concurrent_per_account: usize,
@@ -39,15 +35,14 @@ struct AdminSettingsData {
}
impl AdminSettingsData {
fn from_config(config: &AppConfig) -> Self {
fn from_settings(settings: &SettingsSnapshot) -> Self {
Self {
model_aliases: config.model_aliases.clone(),
model_account_routes: config.model_account_routes.clone(),
refresh_margin_seconds: config.auth.refresh_margin_seconds,
refresh_concurrency: config.auth.refresh_concurrency,
max_concurrent_per_account: config.auth.max_concurrent_per_account,
request_interval_ms: config.auth.request_interval_ms,
rotation_strategy: config.auth.rotation_strategy.clone(),
model_aliases: settings.model_aliases.clone(),
refresh_margin_seconds: settings.refresh_margin_seconds,
refresh_concurrency: settings.refresh_concurrency,
max_concurrent_per_account: settings.max_concurrent_per_account,
request_interval_ms: settings.request_interval_ms,
rotation_strategy: settings.rotation_strategy.clone(),
}
}
}
@@ -60,7 +55,7 @@ pub(crate) async fn settings(
let config = state.services.settings.current();
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AdminSettingsData::from_config(&config)),
AdminEnvelope::ok(AdminSettingsData::from_settings(&config)),
))
}
@@ -77,13 +72,10 @@ pub(crate) async fn update_settings(
.update(patch)
.await
.map_err(settings_service_error)?;
state.services.models.update_config(ModelConfig {
model_aliases: config.model_aliases.clone(),
});
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(AdminSettingsData::from_config(&config)),
AdminEnvelope::ok(AdminSettingsData::from_settings(&config)),
))
}
@@ -141,7 +133,7 @@ pub(crate) async fn delete_admin_api_key(
))
}
fn parse_settings_patch(body: &[u8]) -> Result<AdminSettingsPatch, AdminError> {
fn parse_settings_patch(body: &[u8]) -> Result<SettingsPatch, AdminError> {
let value = serde_json::from_slice::<Value>(body)
.map_err(|_| AdminError::malformed_json("Malformed JSON body"))?;
let object = value
@@ -160,15 +152,12 @@ fn parse_settings_patch(body: &[u8]) -> Result<AdminSettingsPatch, AdminError> {
serde_json::from_value(value).map_err(|_| AdminError::bad_request("Invalid settings payload"))
}
fn settings_service_error(error: RuntimeSettingsError) -> AdminError {
fn settings_service_error(error: SettingsError) -> AdminError {
match error {
RuntimeSettingsError::InvalidField(SettingsServiceError::InvalidField {
field,
message,
}) => AdminError::bad_request(format!("Invalid setting {field}: {message}")),
RuntimeSettingsError::Database(_)
| RuntimeSettingsError::Json(_)
| RuntimeSettingsError::StoredField { .. } => {
SettingsError::InvalidField(SettingsValidationError::InvalidField { field, message }) => {
AdminError::bad_request(format!("Invalid setting {field}: {message}"))
}
SettingsError::Database(_) | SettingsError::Json(_) | SettingsError::StoredField { .. } => {
AdminError::settings_persist_failed("Failed to persist settings")
}
}
+158
View File
@@ -0,0 +1,158 @@
//! 管理端系统版本、自更新与重启 HTTP 处理器。
use std::{convert::Infallible, time::Duration};
use axum::{
extract::{Json, Query, State},
http::StatusCode,
response::{
sse::{Event, KeepAlive, Sse},
IntoResponse,
},
};
use futures::Stream;
use serde::Deserialize;
use crate::{
api::admin::{
response::{AdminEnvelope, AdminError, AdminResponse},
session::AdminAuth,
},
api::AppState,
update::{
service::{self, RestartAction},
state::operation_id,
types::UpdateError,
},
};
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct UpdateDetailQuery {
refresh: Option<bool>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct UpdateRequest {
target_version: String,
}
pub(crate) async fn version(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(service::version_data().await),
))
}
pub(crate) async fn update_detail(
_auth: AdminAuth,
Query(query): Query<UpdateDetailQuery>,
) -> Result<impl IntoResponse, AdminError> {
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(service::update_detail(query.refresh.unwrap_or(false)).await),
))
}
pub(crate) async fn update_event_stream(
State(state): State<AppState>,
_auth: AdminAuth,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, AdminError> {
let receiver = service::subscribe_update_events();
let shutdown = state.services.process_control.subscribe_shutdown();
let stream = futures::stream::unfold(
(receiver, shutdown, false),
|(mut receiver, mut shutdown, close_after_send)| async move {
if close_after_send {
return None;
}
loop {
tokio::select! {
_ = shutdown.recv() => return None,
received = receiver.recv() => match received {
Ok(message) => {
let close_after_send = message.is_terminal();
let id = message.id().to_string();
let data = serde_json::to_string(&message)
.unwrap_or_else(|_| "{}".to_string());
let event = Event::default().event("update").id(id).data(data);
return Some((Ok(event), (receiver, shutdown, close_after_send)));
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
}
}
}
},
);
Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
}
pub(crate) async fn perform_update(
_auth: AdminAuth,
payload: Option<Json<UpdateRequest>>,
) -> Result<impl IntoResponse, AdminError> {
let target = payload.map(|Json(payload)| payload.target_version);
let result = service::perform_update(target)
.await
.map_err(update_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(result),
))
}
pub(crate) async fn update_status(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
let status = service::update_status().map_err(update_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(status),
))
}
pub(crate) async fn rollback(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
let operation_id = service::rollback().await.map_err(update_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(serde_json::json!({
"message": "回滚完成,请重启服务。",
"needRestart": true,
"operationId": operation_id,
})),
))
}
pub(crate) async fn restart(
State(state): State<AppState>,
_auth: AdminAuth,
) -> Result<impl IntoResponse, AdminError> {
let plan = service::restart_plan().map_err(update_error)?;
let message = plan.message;
let process_control = state.services.process_control.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(500)).await;
match plan.action {
RestartAction::Exec(executable_path) => {
process_control.request_restart(executable_path);
}
RestartAction::Shutdown => process_control.request_shutdown(),
}
});
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(serde_json::json!({
"message": message,
"operationId": operation_id("restart"),
})),
))
}
fn update_error(error: UpdateError) -> AdminError {
match error {
UpdateError::BadRequest(message) => AdminError::bad_request(message),
UpdateError::Conflict(message) => AdminError::conflict(message),
UpdateError::BadGateway(message) => AdminError::bad_gateway(message),
UpdateError::Internal(message) => AdminError::internal(message),
}
}
@@ -8,46 +8,39 @@ use axum::{
response::IntoResponse,
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
admin::monitoring::{
usage_record_model::{
metadata_nonnegative_i64, metadata_string, UsageRecord, UsageRecordLevel,
},
usage_record_service::{
usage_record_cost_details, AdminUsageRecordError, AdminUsageRecordFilter,
},
usage_record_store::{
UsageRecordBreakdown, UsageRecordEndpointSource, UsageRecordModelSource,
UsageRecordSummary, UsageRecordTrendPoint,
},
},
admin::{
auth::session::AdminAuth,
api::admin::{
response::{AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse},
session::AdminAuth,
},
api::AppState,
infra::{
format::{
format_compact_number, format_cost, format_duration_ms, format_duration_ms_f64,
format_multiplier, format_number, format_percent, format_rate, format_token_price,
format_tokens,
format_multiplier, format_number, format_percent, format_token_price, format_tokens,
},
json::{clamp_limit, clamp_page, NumberedPage, Page},
json::{clamp_limit, clamp_page, NumberedPage},
time::china_datetime,
},
runtime::state::AppState,
telemetry::{
usage::query::{
usage_record_cost_details, UsageQueryError, UsageQueryFilter, UsageRecordBreakdown,
UsageRecordEndpointSource, UsageRecordModelSource, UsageRecordSummary,
UsageRecordTrendPoint,
},
usage::types::{metadata_string, UsageRecord},
},
};
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct UsageRecordsQuery {
cursor: Option<String>,
limit: Option<u32>,
page: Option<u32>,
page_size: Option<u32>,
kind: Option<String>,
level: Option<String>,
client_api_key_id: Option<String>,
provider: Option<String>,
request_id: Option<String>,
account_id: Option<String>,
route: Option<String>,
@@ -55,8 +48,6 @@ pub(crate) struct UsageRecordsQuery {
status_code: Option<i64>,
transport: Option<String>,
attempt_index: Option<i64>,
upstream_status_code: Option<i64>,
failure_class: Option<String>,
response_id: Option<String>,
upstream_request_id: Option<String>,
search: Option<String>,
@@ -147,8 +138,6 @@ pub(crate) struct UsageRecordCostDetailsData {
#[serde(rename_all = "camelCase")]
struct UsageRecordSummaryData {
total_requests: String,
error_requests: String,
error_rate: String,
input_tokens: String,
output_tokens: String,
cached_tokens: String,
@@ -183,8 +172,6 @@ struct UsageRecordTrendPointData {
input_tokens_value: u64,
output_tokens: String,
output_tokens_value: u64,
cache_creation_tokens: String,
cache_creation_tokens_value: u64,
cached_tokens: String,
cached_tokens_value: u64,
total_tokens: String,
@@ -203,45 +190,14 @@ pub(crate) async fn usage_records(
_auth: AdminAuth,
Query(query): Query<UsageRecordsQuery>,
) -> Result<impl IntoResponse, AdminError> {
let limit = clamp_limit(query.page_size.or(query.limit).unwrap_or(50));
let page = query.page;
let use_numbered_page = page.is_some() || query.page_size.is_some();
let cursor = query.cursor.clone();
let page = clamp_page(query.page.unwrap_or(1));
let page_size = clamp_limit(query.page_size.unwrap_or(50));
let filter = filter_from_query(query)?;
if use_numbered_page {
return match state
.services
.usage_records
.list_page(clamp_page(page.unwrap_or(1)), limit, filter)
.await
{
Ok(page) => {
let account_emails = state
.services
.usage_records
.account_email_map(&page.items)
.await
.map_err(|error| log_error(&error))?;
let page = NumberedPage {
items: usage_record_items(page.items, &account_emails),
total: page.total,
page: page.page,
page_size: page.page_size,
};
Ok(AdminResponse::new(
StatusCode::OK,
AdminPageEnvelope::numbered(page),
))
}
Err(error) => Err(log_error(&error)),
};
}
match state
.services
.usage_records
.list(cursor, limit, filter)
.list_page(page, page_size, filter)
.await
{
Ok(page) => {
@@ -251,13 +207,15 @@ pub(crate) async fn usage_records(
.account_email_map(&page.items)
.await
.map_err(|error| log_error(&error))?;
let page = Page {
let page = NumberedPage {
items: usage_record_items(page.items, &account_emails),
next_cursor: page.next_cursor,
total: page.total,
page: page.page,
page_size: page.page_size,
};
Ok(AdminResponse::new(
StatusCode::OK,
AdminPageEnvelope::ok(page, limit),
AdminPageEnvelope::ok(page),
))
}
Err(error) => Err(log_error(&error)),
@@ -401,10 +359,11 @@ pub(crate) async fn clear_usage_records(
}
}
fn filter_from_query(query: UsageRecordsQuery) -> Result<AdminUsageRecordFilter, AdminError> {
Ok(AdminUsageRecordFilter {
fn filter_from_query(query: UsageRecordsQuery) -> Result<UsageQueryFilter, AdminError> {
Ok(UsageQueryFilter {
kind: non_empty(query.kind),
level: level_from_query(query.level).map_err(AdminError::bad_request)?,
client_api_key_id: non_empty(query.client_api_key_id),
provider: non_empty(query.provider),
request_id: non_empty(query.request_id),
account_id: non_empty(query.account_id),
route: non_empty(query.route),
@@ -412,8 +371,6 @@ fn filter_from_query(query: UsageRecordsQuery) -> Result<AdminUsageRecordFilter,
status_code: query.status_code,
transport: non_empty(query.transport),
attempt_index: query.attempt_index,
upstream_status_code: query.upstream_status_code,
failure_class: non_empty(query.failure_class),
response_id: non_empty(query.response_id),
upstream_request_id: non_empty(query.upstream_request_id),
search: non_empty(query.search),
@@ -457,29 +414,12 @@ fn optional_datetime(
.map_err(|_| AdminError::invalid_time_range("Invalid time range"))
}
fn log_error(error: &AdminUsageRecordError) -> AdminError {
fn log_error(error: &UsageQueryError) -> AdminError {
match error {
AdminUsageRecordError::List
| AdminUsageRecordError::Get
| AdminUsageRecordError::Clear
| AdminUsageRecordError::Append
| AdminUsageRecordError::Trim => AdminError::internal(error.to_string()),
AdminUsageRecordError::Accounts => {
AdminError::usage_record_accounts_failed(error.to_string())
UsageQueryError::List | UsageQueryError::Get | UsageQueryError::Clear => {
AdminError::internal(error.to_string())
}
}
}
fn level_from_query(value: Option<String>) -> Result<Option<UsageRecordLevel>, String> {
let Some(value) = non_empty(value) else {
return Ok(None);
};
match value.as_str() {
"debug" => Ok(Some(UsageRecordLevel::Debug)),
"info" => Ok(Some(UsageRecordLevel::Info)),
"warn" => Ok(Some(UsageRecordLevel::Warn)),
"error" => Ok(Some(UsageRecordLevel::Error)),
other => Err(format!("Unsupported log level: {other}")),
UsageQueryError::Accounts => AdminError::usage_record_accounts_failed(error.to_string()),
}
}
@@ -503,10 +443,8 @@ fn usage_record_data(
record: UsageRecord,
account_emails: &HashMap<String, String>,
) -> UsageRecordData {
let account_email = record
.account_id
.as_deref()
.and_then(|account_id| account_emails.get(account_id))
let account_email = account_emails
.get(&record.account_id)
.cloned()
.or_else(|| metadata_string(&record.metadata, &["accountEmail", "account_email"]));
let (requested_model, upstream_model) = usage_record_models(&record);
@@ -514,17 +452,9 @@ fn usage_record_data(
let user_agent = metadata_string(&record.metadata, &["userAgent", "user_agent"]);
let reasoning_effort =
metadata_string(&record.metadata, &["reasoningEffort", "reasoning_effort"]);
let token_details = usage_token_details(&record.metadata);
let token_details = usage_token_details(&record);
let cost_details = usage_cost_details(&record, upstream_model.as_deref(), &token_details);
let first_token_latency_ms = metadata_nonnegative_i64(
&record.metadata,
&[
"firstTokenMs",
"first_token_ms",
"firstTokenLatencyMs",
"first_token_latency_ms",
],
);
let first_token_latency_ms = record.first_token_ms;
let first_token_latency_ms_display = format_duration_ms(first_token_latency_ms);
let latency_ms_display = format_duration_ms(record.latency_ms);
@@ -545,28 +475,39 @@ fn usage_record_data(
}
}
fn usage_record_models(record: &UsageRecord) -> (Option<String>, Option<String>) {
let stored_model = record
.model
.as_deref()
.map(str::trim)
.filter(|model| !model.is_empty());
let requested_model = metadata_string(&record.metadata, &["requestedModel"])
.or_else(|| stored_model.map(ToString::to_string));
let upstream_model = metadata_string(&record.metadata, &["upstreamModel"])
pub(crate) fn usage_record_models(record: &UsageRecord) -> (Option<String>, Option<String>) {
let stored_model = record.model.trim();
let requested_model = record
.requested_model
.clone()
.or_else(|| (!stored_model.is_empty()).then(|| stored_model.to_string()));
let upstream_model = record
.upstream_model
.clone()
.or_else(|| requested_model.clone())
.or_else(|| stored_model.map(ToString::to_string));
.or_else(|| (!stored_model.is_empty()).then(|| stored_model.to_string()));
(requested_model, upstream_model)
}
fn usage_token_details(metadata: &Value) -> UsageRecordTokenDetailsData {
let input_tokens = metadata_usage_u64(metadata, "inputTokens");
let output_tokens = metadata_usage_u64(metadata, "outputTokens");
let cached_tokens = metadata_usage_u64(metadata, "cachedTokens");
let reasoning_tokens = metadata_usage_u64(metadata, "reasoningTokens");
let total_tokens =
metadata_usage_u64(metadata, "totalTokens").max(input_tokens.saturating_add(output_tokens));
pub(crate) fn usage_token_details(record: &UsageRecord) -> UsageRecordTokenDetailsData {
let input_tokens: u64 = record
.input_tokens
.and_then(|value| value.try_into().ok())
.unwrap_or(0);
let output_tokens: u64 = record
.output_tokens
.and_then(|value| value.try_into().ok())
.unwrap_or(0);
let cached_tokens: u64 = record
.cached_tokens
.and_then(|value| value.try_into().ok())
.unwrap_or(0);
let reasoning_tokens: u64 = record
.reasoning_tokens
.and_then(|value| value.try_into().ok())
.unwrap_or(0);
let total_tokens = input_tokens.saturating_add(output_tokens);
UsageRecordTokenDetailsData {
input_tokens,
@@ -582,7 +523,7 @@ fn usage_token_details(metadata: &Value) -> UsageRecordTokenDetailsData {
}
}
fn usage_cost_details(
pub(crate) fn usage_cost_details(
record: &UsageRecord,
upstream_model: Option<&str>,
tokens: &UsageRecordTokenDetailsData,
@@ -619,32 +560,10 @@ fn usage_cost_details(
})
}
fn metadata_usage_u64(metadata: &Value, key: &str) -> u64 {
metadata
.get("usage")
.and_then(|usage| metadata_u64_at(usage, key))
.or_else(|| metadata_u64_at(metadata, key))
.unwrap_or_default()
}
fn metadata_u64_at(value: &Value, key: &str) -> Option<u64> {
value
.get(key)
.and_then(|value| value.as_u64().or_else(|| value.as_i64()?.try_into().ok()))
}
impl From<UsageRecordSummary> for UsageRecordSummaryData {
fn from(summary: UsageRecordSummary) -> Self {
Self {
total_requests: format_compact_number(summary.total_requests),
error_requests: format_compact_number(summary.error_requests),
error_rate: if summary.total_requests > 0 {
format_rate(Some(
summary.error_requests as f64 / summary.total_requests as f64,
))
} else {
"—".to_string()
},
input_tokens: format_tokens(summary.input_tokens),
output_tokens: format_tokens(summary.output_tokens),
cached_tokens: format_tokens(summary.cached_tokens),
@@ -656,7 +575,7 @@ impl From<UsageRecordSummary> for UsageRecordSummaryData {
impl From<UsageRecordBreakdown> for UsageRecordBreakdownData {
fn from(item: UsageRecordBreakdown) -> Self {
UsageRecordBreakdownData::from_item(item, 0)
Self::from_item(item, 0)
}
}
@@ -694,10 +613,7 @@ fn usage_record_breakdown_items(items: Vec<UsageRecordBreakdown>) -> Vec<UsageRe
impl From<UsageRecordTrendPoint> for UsageRecordTrendPointData {
fn from(point: UsageRecordTrendPoint) -> Self {
let prompt_tokens = point
.input_tokens
.saturating_add(point.cache_creation_tokens)
.saturating_add(point.cached_tokens);
let prompt_tokens = point.input_tokens.saturating_add(point.cached_tokens);
let cache_hit_rate_value = if prompt_tokens > 0 {
(point.cached_tokens as f64 / prompt_tokens as f64 * 100.0).round()
} else {
@@ -709,8 +625,6 @@ impl From<UsageRecordTrendPoint> for UsageRecordTrendPointData {
input_tokens_value: point.input_tokens,
output_tokens: format_tokens(point.output_tokens),
output_tokens_value: point.output_tokens,
cache_creation_tokens: format_tokens(point.cache_creation_tokens),
cache_creation_tokens_value: point.cache_creation_tokens,
cached_tokens: format_tokens(point.cached_tokens),
cached_tokens_value: point.cached_tokens,
total_tokens: format_tokens(point.total_tokens),
@@ -4,7 +4,7 @@
use axum::http::{header::AUTHORIZATION, HeaderMap};
use crate::runtime::state::AppState;
use crate::api::AppState;
/// client API key 鉴权失败原因。
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -56,11 +56,16 @@ pub fn bearer_client_api_key(headers: &HeaderMap) -> Result<&str, ClientApiKeyAu
/// 校验客户端 API key。
pub async fn authorize_client_api_key(state: &AppState, headers: &HeaderMap) -> bool {
authorized_client_api_key_id(state, headers).await.is_some()
}
/// 校验客户端 API key,成功时返回用于事实归因的稳定 ID。
pub async fn authorized_client_api_key_id(state: &AppState, headers: &HeaderMap) -> Option<String> {
match authorize_client_api_key_result(state, headers).await {
Ok(()) => true,
Ok(key_id) => Some(key_id),
Err(error) => {
log_client_api_key_auth_failure(&error);
false
None
}
}
}
@@ -69,7 +74,7 @@ pub async fn authorize_client_api_key(state: &AppState, headers: &HeaderMap) ->
pub async fn authorize_client_api_key_result(
state: &AppState,
headers: &HeaderMap,
) -> Result<(), ClientApiKeyAuthError> {
) -> Result<String, ClientApiKeyAuthError> {
let api_key = bearer_client_api_key(headers)?;
let verified = state
.services
@@ -79,9 +84,7 @@ pub async fn authorize_client_api_key_result(
.map_err(|error| ClientApiKeyAuthError::StoreUnavailable {
message: error.to_string(),
})?;
verified
.then_some(())
.ok_or(ClientApiKeyAuthError::InvalidKey)
verified.ok_or(ClientApiKeyAuthError::InvalidKey)
}
fn log_client_api_key_auth_failure(error: &ClientApiKeyAuthError) {
@@ -94,7 +97,7 @@ fn log_client_api_key_auth_failure(error: &ClientApiKeyAuthError) {
);
}
_ => {
tracing::warn!(
tracing::info!(
auth_failure = error.reason(),
"client api key authorization failed"
);
@@ -7,9 +7,10 @@ use axum::{
};
use serde_json::{json, Value};
use crate::proxy::{
dispatch::{errors::DispatchFailureClass, responses::errors::ResponseDispatchError},
openai::responses::response_failed_sse_event as encode_response_failed_sse_event,
use crate::{
dispatch::errors::{DispatchFailureClass, ResponseDispatchError},
upstream::openai::protocol::sse::response_failed_sse_event as encode_response_failed_sse_event,
upstream::openai::transport::CodexClientError,
};
const NO_ACTIVE_UPSTREAM_ACCOUNT_MESSAGE: &str = "No active upstream account is available";
@@ -18,29 +19,26 @@ const NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE: &str =
const UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE: &str = "Upstream Codex request failed";
const INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE: &str = "Invalid upstream Codex response";
const UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE: &str = "Upstream Codex response failed";
const HISTORY_UNAVAILABLE_MESSAGE: &str = "Previous response context is unavailable. Start a new conversation or resend the complete input without previous_response_id.";
const CONTINUATION_BUSY_MESSAGE: &str =
"The previous response connection is busy. Retry the request.";
#[derive(Clone, Copy)]
pub enum ResponseDispatchMessageStyle {
enum ResponseDispatchMessageStyle {
Standard,
ResponsesStream,
}
#[derive(Clone, Copy)]
pub enum ResponseDispatchStatusMode {
UpstreamFailureStatus,
Client,
struct DispatchHttpError {
status: StatusCode,
message: String,
}
pub struct DispatchHttpError {
pub status: StatusCode,
pub message: String,
}
pub struct OpenAiErrorDetails {
pub status: StatusCode,
pub message: String,
pub error_type: &'static str,
pub code: &'static str,
struct OpenAiErrorDetails {
status: StatusCode,
message: String,
error_type: &'static str,
code: &'static str,
}
#[derive(Debug, Clone, Copy)]
@@ -49,9 +47,9 @@ struct OpenAiErrorKind {
code: &'static str,
}
const UPSTREAM_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
const NO_AVAILABLE_ACCOUNTS_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "server_error",
code: "upstream_unavailable",
code: "no_available_accounts",
};
const UPSTREAM_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "server_error",
@@ -73,6 +71,14 @@ const MODEL_NOT_FOUND_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "invalid_request_error",
code: "model_not_found",
};
const HISTORY_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "invalid_request_error",
code: "previous_response_unavailable",
};
const CONTINUATION_BUSY_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "server_error",
code: "continuation_connection_busy",
};
const INVALID_UPSTREAM_RESPONSE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "server_error",
code: "invalid_upstream_response",
@@ -81,6 +87,10 @@ const CODEX_API_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "server_error",
code: "codex_api_error",
};
const CODEX_CLIENT_ERROR: OpenAiErrorKind = OpenAiErrorKind {
error_type: "invalid_request_error",
code: "codex_client_error",
};
/// OpenAI 兼容错误响应。
pub fn openai_error_response(
@@ -102,62 +112,43 @@ pub fn openai_error_response(
}
pub fn responses_dispatch_error_response(error: ResponseDispatchError) -> Response {
match error {
ResponseDispatchError::NoActiveAccount => {
responses_no_available_accounts_response().into_response()
}
error => response_dispatch_openai_error_response(&error),
}
responses_dispatch_error_response_ref(&error)
}
pub fn responses_compact_dispatch_error_response(error: ResponseDispatchError) -> Response {
pub fn responses_dispatch_error_response_ref(error: &ResponseDispatchError) -> Response {
if let ResponseDispatchError::Upstream(CodexClientError::Upstream { status, body, .. }) = error
{
if status.is_client_error() {
if let Ok(body) = serde_json::from_str::<Value>(body) {
return (*status, Json(body)).into_response();
}
}
}
match error {
ResponseDispatchError::NoActiveAccount => {
responses_no_available_accounts_response().into_response()
}
error => {
let error = response_dispatch_http_error(
&error,
ResponseDispatchStatusMode::Client,
ResponseDispatchMessageStyle::Standard,
);
responses_error_response(error.status, &error.message).into_response()
}
error => response_dispatch_openai_error_response(error),
}
}
pub fn responses_stream_dispatch_failed_sse_event(error: &ResponseDispatchError) -> String {
let http_error = response_dispatch_http_error(
error,
ResponseDispatchStatusMode::UpstreamFailureStatus,
ResponseDispatchMessageStyle::ResponsesStream,
);
let http_error =
response_dispatch_http_error(error, ResponseDispatchMessageStyle::ResponsesStream);
let kind = response_dispatch_error_kind(error);
responses_failed_sse_event(kind, &http_error.message)
}
fn responses_no_available_accounts_response() -> (StatusCode, Json<Value>) {
let kind = NO_AVAILABLE_ACCOUNTS_ERROR;
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"error": {
"type": "server_error",
"code": "no_available_accounts",
"message": NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE,
}
})),
)
}
fn responses_error_response(status: StatusCode, message: &str) -> (StatusCode, Json<Value>) {
let kind = responses_error_kind_for_status(status);
(
status,
Json(json!({
"error": {
"type": kind.error_type,
"code": kind.code,
"message": message,
"message": NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE,
}
})),
)
@@ -208,11 +199,8 @@ fn invalid_openai_request_response(message: &str) -> (StatusCode, Json<Value>) {
}
fn response_dispatch_openai_error(error: &ResponseDispatchError) -> OpenAiErrorDetails {
let DispatchHttpError { status, message } = response_dispatch_http_error(
error,
ResponseDispatchStatusMode::UpstreamFailureStatus,
ResponseDispatchMessageStyle::Standard,
);
let DispatchHttpError { status, message } =
response_dispatch_http_error(error, ResponseDispatchMessageStyle::Standard);
let kind = response_dispatch_error_kind(error);
OpenAiErrorDetails {
@@ -226,17 +214,16 @@ fn response_dispatch_openai_error(error: &ResponseDispatchError) -> OpenAiErrorD
fn response_dispatch_error_kind(error: &ResponseDispatchError) -> OpenAiErrorKind {
dispatch_failure_openai_error_kind(
error.metadata().failure_class,
status_from_u16(error.http_status_code(), StatusCode::BAD_GATEWAY),
status_from_u16(error.client_http_status_code(), StatusCode::BAD_GATEWAY),
)
}
pub fn response_dispatch_http_error(
fn response_dispatch_http_error(
error: &ResponseDispatchError,
status_mode: ResponseDispatchStatusMode,
message_style: ResponseDispatchMessageStyle,
) -> DispatchHttpError {
DispatchHttpError {
status: response_dispatch_status(error, status_mode),
status: status_from_u16(error.client_http_status_code(), StatusCode::BAD_GATEWAY),
message: response_dispatch_message(error, message_style),
}
}
@@ -249,27 +236,12 @@ fn responses_error_kind_for_status(status: StatusCode) -> OpenAiErrorKind {
} else if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
AUTHENTICATION_ERROR
} else if status.is_client_error() {
OpenAiErrorKind {
error_type: "invalid_request_error",
code: "codex_api_error",
}
CODEX_CLIENT_ERROR
} else {
CODEX_API_ERROR
}
}
fn response_dispatch_status(
error: &ResponseDispatchError,
mode: ResponseDispatchStatusMode,
) -> StatusCode {
match (mode, error) {
(ResponseDispatchStatusMode::UpstreamFailureStatus, ResponseDispatchError::Upstream(_)) => {
StatusCode::BAD_GATEWAY
}
_ => status_from_u16(error.http_status_code(), StatusCode::BAD_GATEWAY),
}
}
fn response_dispatch_message(
error: &ResponseDispatchError,
style: ResponseDispatchMessageStyle,
@@ -320,6 +292,8 @@ fn dispatch_failure_message(
INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE.to_owned()
}
DispatchFailureClass::ResponseFailed => UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE.to_owned(),
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_MESSAGE.to_owned(),
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_MESSAGE.to_owned(),
_ => UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE.to_owned(),
}
}
@@ -329,20 +303,28 @@ fn dispatch_failure_openai_error_kind(
status: StatusCode,
) -> OpenAiErrorKind {
match failure_class {
DispatchFailureClass::NoAvailableAccounts => UPSTREAM_UNAVAILABLE_ERROR,
DispatchFailureClass::NoAvailableAccounts => NO_AVAILABLE_ACCOUNTS_ERROR,
DispatchFailureClass::QuotaExhausted => INSUFFICIENT_QUOTA_ERROR,
DispatchFailureClass::RateLimited => RATE_LIMIT_ERROR,
DispatchFailureClass::Expired
| DispatchFailureClass::Disabled
| DispatchFailureClass::Banned => AUTHENTICATION_ERROR,
DispatchFailureClass::ModelUnsupported => MODEL_NOT_FOUND_ERROR,
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_ERROR,
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_ERROR,
DispatchFailureClass::InvalidSse
| DispatchFailureClass::MissingCompleted
| DispatchFailureClass::EmptyUpstreamResponse => INVALID_UPSTREAM_RESPONSE_ERROR,
DispatchFailureClass::ResponseFailed => responses_error_kind_for_status(status),
DispatchFailureClass::Upstream
| DispatchFailureClass::CloudflareChallenge
| DispatchFailureClass::CloudflarePathBlocked => UPSTREAM_ERROR,
DispatchFailureClass::Upstream if status.is_client_error() => {
responses_error_kind_for_status(status)
}
DispatchFailureClass::Upstream | DispatchFailureClass::UpstreamUnavailable => {
UPSTREAM_ERROR
}
DispatchFailureClass::CloudflareChallenge | DispatchFailureClass::CloudflarePathBlocked => {
UPSTREAM_ERROR
}
}
}
@@ -1,7 +1,8 @@
//! OpenAI 兼容 API 模块。
pub mod auth;
pub mod errors;
pub mod models;
pub mod responses;
pub mod routes;
pub mod router;
pub mod sse;
@@ -9,8 +9,7 @@ use axum::{
use serde_json::{json, Value};
use crate::{
proxy::auth::authorize_client_api_key, runtime::state::AppState,
upstream::models::catalog::ModelCatalog,
api::client::auth::authorize_client_api_key, api::AppState, models::catalog::ModelCatalog,
};
use super::errors::{missing_client_api_key_response, model_not_found_response};
@@ -8,29 +8,26 @@ use axum::{
response::{IntoResponse, Response},
Extension, Json,
};
use serde_json::{json, Map, Value};
use serde_json::{Map, Value};
use crate::{
http::middleware::request_id::{ClientIp, RequestId},
proxy::{
auth::authorize_client_api_key,
dispatch::responses::{errors::ResponseDispatchError, service::ResponseDispatchStream},
},
runtime::state::AppState,
upstream::protocol::{
api::middleware::request_id::{ClientIp, RequestId},
api::AppState,
dispatch::{errors::ResponseDispatchError, service::ResponseDispatchStream},
upstream::openai::protocol::{
responses::{CodexCompactRequest, CodexResponsesRequest},
sse::{encode_sse_event, sse_body_has_done},
sse::sse_body_has_done,
},
};
use super::{
errors::{
invalid_responses_request_response, missing_client_api_key_response,
model_not_found_response, responses_compact_dispatch_error_response,
responses_dispatch_error_response, responses_stream_dispatch_failed_sse_event,
model_not_found_response, responses_dispatch_error_response,
responses_dispatch_error_response_ref, responses_stream_dispatch_failed_sse_event,
},
models::model_catalog_for_state,
sse::{done_sse_frame, event_stream_response as sse_event_stream_response, SseResponseOptions},
sse::{done_sse_frame, event_stream_response as sse_event_stream_response},
};
const OPENAI_SUBAGENT_HEADER: &str = "x-openai-subagent";
@@ -104,10 +101,43 @@ pub fn build_codex_request(
None => {}
}
apply_context_header_fields(&mut request, headers);
apply_identity_fields(&mut request, headers);
request.client_user_agent = user_agent_from_headers(headers);
request
}
fn apply_identity_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) {
request.client_conversation_id = request.prompt_cache_key().map(ToString::to_string);
request.client_session_id = identity_context_string(request.body(), "session_id")
.or_else(|| header_string(headers, "session-id"));
request.client_thread_id = identity_context_string(request.body(), "thread_id")
.or_else(|| header_string(headers, "thread-id"));
request.client_request_id = identity_context_string(request.body(), "x-client-request-id")
.or_else(|| header_string(headers, "x-client-request-id"));
request.client_turn_id = identity_context_string(request.body(), "turn_id")
.or_else(|| header_string(headers, "x-codex-turn-id"));
request.codex_window_id = request.codex_window_id.take().or_else(|| {
identity_context_string(request.body(), "x-codex-window-id")
.or_else(|| header_string(headers, "x-codex-window-id"))
});
request.parent_thread_id = request.parent_thread_id.take().or_else(|| {
identity_context_string(request.body(), "x-codex-parent-thread-id")
.or_else(|| header_string(headers, "x-codex-parent-thread-id"))
});
}
fn identity_context_string(body: &Map<String, Value>, key: &str) -> Option<String> {
body_context_string(body, key).or_else(|| {
body.get("client_metadata")
.and_then(Value::as_object)
.and_then(|metadata| metadata.get(key))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
})
}
/// 从客户端原始 Responses JSON body 构造 Codex compact 请求。
///
/// compact 端点只返回一次性 JSON(代理按非流式读取),故剥离 `stream` 这一
@@ -118,6 +148,18 @@ pub fn build_compact_request(
mut body: Map<String, Value>,
headers: &HeaderMap,
) -> CodexCompactRequest {
let client_session_id = identity_context_string(&body, "session_id")
.or_else(|| header_string(headers, "session-id"));
let client_thread_id =
identity_context_string(&body, "thread_id").or_else(|| header_string(headers, "thread-id"));
let client_request_id = identity_context_string(&body, "x-client-request-id")
.or_else(|| header_string(headers, "x-client-request-id"));
let client_turn_id = identity_context_string(&body, "turn_id")
.or_else(|| header_string(headers, "x-codex-turn-id"));
let client_window_id = identity_context_string(&body, "x-codex-window-id")
.or_else(|| header_string(headers, "x-codex-window-id"));
let client_parent_thread_id = identity_context_string(&body, "x-codex-parent-thread-id")
.or_else(|| header_string(headers, "x-codex-parent-thread-id"));
body.remove(COMPACT_STREAM_KEY);
for key in TRANSPORT_ONLY_KEYS {
body.remove(key);
@@ -126,6 +168,13 @@ pub fn build_compact_request(
body,
client_ip: None,
client_user_agent: user_agent_from_headers(headers),
client_api_key_id: None,
client_session_id,
client_thread_id,
client_request_id,
client_turn_id,
client_window_id,
client_parent_thread_id,
}
}
@@ -193,41 +242,6 @@ impl ContextField {
}
}
/// 编码 OpenAI Responses `response.failed` SSE 事件。
pub fn response_failed_sse_event(error_type: &str, code: &str, message: &str) -> String {
response_failed_sse_event_with_id(None, error_type, code, message)
}
/// 使用指定 response id 编码 OpenAI Responses `response.failed` SSE 事件。
pub fn response_failed_sse_event_with_id(
response_id: Option<&str>,
error_type: &str,
code: &str,
message: &str,
) -> String {
let error = json!({
"type": error_type,
"code": code,
"message": message,
});
let response_id = response_id
.filter(|value| !value.trim().is_empty())
.map_or_else(
|| format!("resp_proxy_{}", uuid::Uuid::new_v4().simple()),
ToString::to_string,
);
let data = json!({
"type": "response.failed",
"response": {
"id": response_id,
"status": "failed",
"error": error,
},
"error": error,
});
encode_sse_event("response.failed", &data.to_string())
}
// ====================================================================
// HTTP 处理器
// ====================================================================
@@ -243,7 +257,7 @@ pub async fn responses(
handle_responses(
state,
request_id,
client_ip_string(client_ip),
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string()),
headers,
body,
"/v1/responses",
@@ -263,7 +277,7 @@ pub async fn review_responses(
handle_responses(
state,
request_id,
client_ip_string(client_ip),
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string()),
headers,
body,
"/v1/responses/review",
@@ -283,18 +297,13 @@ pub async fn compact_responses(
handle_compact_responses(
state,
request_id,
client_ip_string(client_ip),
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string()),
headers,
body,
)
.await
}
/// 从可选的 `ClientIp` extension 提取 IP 字符串。
fn client_ip_string(client_ip: Option<Extension<ClientIp>>) -> Option<String> {
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string())
}
async fn handle_responses(
state: AppState,
request_id: RequestId,
@@ -304,9 +313,11 @@ async fn handle_responses(
route: &str,
forced_subagent: Option<&str>,
) -> Response {
if !authorize_client_api_key(&state, &headers).await {
let Some(client_api_key_id) =
crate::api::client::auth::authorized_client_api_key_id(&state, &headers).await
else {
return missing_client_api_key_response().into_response();
}
};
let Some(body) = parse_responses_body(&body) else {
return invalid_responses_request_response().into_response();
@@ -322,6 +333,7 @@ async fn handle_responses(
};
let mut codex_request = build_codex_request(body, &headers, forced_subagent);
codex_request.client_ip = client_ip;
codex_request.client_api_key_id = Some(client_api_key_id);
// 客户端未显式指定 transport 时默认偏好 WebSocket。
if !codex_request.force_http_sse {
codex_request.use_websocket = true;
@@ -346,7 +358,10 @@ async fn handle_responses(
.complete(request_id.as_str(), route, codex_request, &model)
.await
{
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
Ok(result) => apply_safe_response_headers(
(StatusCode::OK, Json(result.body)).into_response(),
result.response_headers,
),
Err(error) => responses_dispatch_error_response(error),
}
}
@@ -358,9 +373,11 @@ async fn handle_compact_responses(
headers: HeaderMap,
body: Bytes,
) -> Response {
if !authorize_client_api_key(&state, &headers).await {
let Some(client_api_key_id) =
crate::api::client::auth::authorized_client_api_key_id(&state, &headers).await
else {
return missing_client_api_key_response().into_response();
}
};
let Some(body) = parse_responses_body(&body) else {
return invalid_responses_request_response().into_response();
@@ -376,6 +393,7 @@ async fn handle_compact_responses(
};
let mut compact_request = build_compact_request(body, &headers);
compact_request.client_ip = client_ip;
compact_request.client_api_key_id = Some(client_api_key_id);
match state
.services
@@ -383,8 +401,11 @@ async fn handle_compact_responses(
.compact(request_id.as_str(), compact_request, &model)
.await
{
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
Err(error) => responses_compact_dispatch_error_response(error),
Ok(result) => apply_safe_response_headers(
(StatusCode::OK, Json(result.body)).into_response(),
result.response_headers,
),
Err(error) => responses_dispatch_error_response(error),
}
}
@@ -451,14 +472,40 @@ fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
fn event_stream_response(mut body: String) -> Response {
ensure_done_sse_frame(&mut body);
sse_event_stream_response(Body::from(body), SseResponseOptions::BASIC)
sse_event_stream_response(Body::from(body))
}
fn live_event_stream_response(stream: ResponseDispatchStream) -> Response {
sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC)
let response = sse_event_stream_response(Body::from_stream(stream.body));
apply_safe_response_headers(response, stream.response_headers)
}
fn apply_safe_response_headers(mut response: Response, headers: Vec<(String, String)>) -> Response {
for (name, value) in headers {
let name = match name.as_str() {
"x-request-id" => axum::http::header::HeaderName::from_static("x-request-id"),
"openai-model" => axum::http::header::HeaderName::from_static("openai-model"),
"x-models-etag" => axum::http::header::HeaderName::from_static("x-models-etag"),
"x-reasoning-included" => {
axum::http::header::HeaderName::from_static("x-reasoning-included")
}
"openai-processing-ms" => {
axum::http::header::HeaderName::from_static("openai-processing-ms")
}
_ => continue,
};
let Ok(value) = axum::http::HeaderValue::from_str(&value) else {
continue;
};
response.headers_mut().insert(name, value);
}
response
}
fn response_dispatch_stream_error_response(error: &ResponseDispatchError) -> Response {
if (400..=499).contains(&error.client_http_status_code()) {
return responses_dispatch_error_response_ref(error);
}
event_stream_response(responses_stream_dispatch_failed_sse_event(error))
}
@@ -1,17 +1,20 @@
//! OpenAI 兼容 API 路由。
use axum::{
extract::DefaultBodyLimit,
routing::{get, post},
Router,
};
use crate::runtime::state::AppState;
use crate::api::AppState;
use super::{
models::{model_catalog, model_detail, model_info, models},
responses::{compact_responses, responses, review_responses},
};
pub const MAX_CLIENT_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
/// 构造 OpenAI 兼容 API 路由。
pub fn router() -> Router<AppState> {
Router::new()
@@ -22,4 +25,5 @@ pub fn router() -> Router<AppState> {
.route("/v1/models/catalog", get(model_catalog))
.route("/v1/models/{model_id}", get(model_detail))
.route("/v1/models/{model_id}/info", get(model_info))
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
}
+38
View File
@@ -0,0 +1,38 @@
//! OpenAI SSE 封装。
//!
//! 提供 SSE 事件编码与流式响应构建工具。
use axum::{
body::Body,
http::{
header::{CACHE_CONTROL, CONTENT_TYPE},
StatusCode,
},
response::{IntoResponse, Response},
};
use crate::upstream::openai::protocol::sse::DONE_SSE_FRAME;
use super::errors::openai_error_response;
/// 编码 OpenAI 流结束标记。
pub fn done_sse_frame() -> &'static str {
DONE_SSE_FRAME
}
pub fn event_stream_response(body: Body) -> Response {
Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "text/event-stream")
.header(CACHE_CONTROL, "no-cache")
.body(body)
.unwrap_or_else(|_| {
openai_error_response(
StatusCode::INTERNAL_SERVER_ERROR,
"Failed to build stream response",
"server_error",
"stream_response_error",
)
.into_response()
})
}
@@ -4,7 +4,7 @@ use std::net::{IpAddr, SocketAddr};
use std::sync::LazyLock;
use axum::{
extract::{connect_info::ConnectInfo, Request, State},
extract::{connect_info::ConnectInfo, Request},
http::{HeaderMap, HeaderValue},
middleware::Next,
response::Response,
@@ -19,7 +19,7 @@ const CF_CONNECTING_IP_HEADER: &str = "cf-connecting-ip";
const REAL_IP_HEADER: &str = "x-real-ip";
const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for";
/// 私有 / 回环 / 唯一本地地址网段,用于在自动模式下从 `X-Forwarded-For` 中跳过内网跳板 IP。
/// 私有 / 回环 / 唯一本地地址网段,用于从 `X-Forwarded-For` 中跳过内网跳板 IP。
static PRIVATE_NETWORKS: LazyLock<Vec<IpNet>> = LazyLock::new(|| {
[
"10.0.0.0/8",
@@ -38,40 +38,6 @@ fn is_private_ip(ip: IpAddr) -> bool {
PRIVATE_NETWORKS.iter().any(|network| network.contains(&ip))
}
/// 可信反向代理配置。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TrustedProxyConfig {
networks: Vec<IpNet>,
}
impl TrustedProxyConfig {
/// 从 IP 或 CIDR 字符串构造可信代理配置。
pub fn from_entries(entries: &[String]) -> Result<Self, TrustedProxyConfigError> {
let networks = entries
.iter()
.map(|entry| parse_trusted_proxy_entry(entry))
.collect::<Result<Vec<_>, _>>()?;
Ok(Self { networks })
}
fn is_trusted(&self, ip: IpAddr) -> bool {
self.networks.iter().any(|network| network.contains(&ip))
}
/// 是否未配置任何可信代理(决定是否启用自动真实 IP 探测)。
fn is_empty(&self) -> bool {
self.networks.is_empty()
}
}
/// 可信代理配置错误。
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum TrustedProxyConfigError {
/// 配置项不是 IP 地址或 CIDR。
#[error("invalid trusted proxy entry `{entry}`; expected an IP address or CIDR network")]
InvalidEntry { entry: String },
}
/// 请求 ID。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestId(String);
@@ -97,7 +63,7 @@ impl RequestId {
}
}
/// 由连接来源和可信代理链解析出的客户端 IP。
/// 由连接来源和转发头解析出的客户端 IP。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ClientIp(String);
@@ -118,17 +84,7 @@ fn valid_request_id(value: &str) -> bool {
/// 为请求附加请求 ID,并在响应头中回写。
pub async fn attach_request_id(mut request: Request, next: Next) -> Response {
attach_request_context(&TrustedProxyConfig::default(), &mut request);
run_with_request_id(request, next).await
}
/// 为请求附加请求上下文,并按可信代理配置解析真实客户端 IP。
pub async fn attach_request_id_with_proxy_config(
State(trusted_proxies): State<TrustedProxyConfig>,
mut request: Request,
next: Next,
) -> Response {
attach_request_context(&trusted_proxies, &mut request);
attach_request_context(&mut request);
run_with_request_id(request, next).await
}
@@ -147,17 +103,16 @@ async fn run_with_request_id(mut request: Request, next: Next) -> Response {
response
}
fn attach_request_context(trusted_proxies: &TrustedProxyConfig, request: &mut Request) {
attach_client_ip_from_connection(trusted_proxies, request);
fn attach_request_context(request: &mut Request) {
attach_client_ip_from_connection(request);
}
fn attach_client_ip_from_connection(trusted_proxies: &TrustedProxyConfig, request: &mut Request) {
fn attach_client_ip_from_connection(request: &mut Request) {
let peer_ip = request
.extensions()
.get::<ConnectInfo<SocketAddr>>()
.map(|ConnectInfo(addr)| addr.ip());
let client_ip =
peer_ip.map(|peer_ip| client_ip_from_peer(peer_ip, request.headers(), trusted_proxies));
let client_ip = peer_ip.map(|peer_ip| client_ip_from_peer(peer_ip, request.headers()));
let Some(client_ip) = client_ip else {
return;
@@ -165,28 +120,12 @@ fn attach_client_ip_from_connection(trusted_proxies: &TrustedProxyConfig, reques
request.extensions_mut().insert(ClientIp(client_ip));
}
fn client_ip_from_peer(
peer_ip: IpAddr,
headers: &HeaderMap,
trusted_proxies: &TrustedProxyConfig,
) -> String {
if trusted_proxies.is_empty() {
// 未配置可信代理:自动模式,按 CF-Connecting-IP → X-Real-IP →
// X-Forwarded-For(首个公网 IP)→ peer IP 的优先级解析真实客户端 IP。
return auto_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string());
}
if trusted_proxies.is_trusted(peer_ip) {
// 已配置可信代理且直连来源可信:只采信转发头。
trusted_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string())
} else {
// 已配置可信代理但直连来源不可信:忽略转发头,使用 peer IP。
peer_ip.to_string()
}
fn client_ip_from_peer(peer_ip: IpAddr, headers: &HeaderMap) -> String {
auto_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string())
}
/// 自动模式下解析转发头中的真实客户端 IP。
/// 解析转发头中的真实客户端 IP。
///
/// 未配置可信代理时使用,对齐常见反代(Cloudflare / Nginx)默认行为:
/// `CF-Connecting-IP` → `X-Real-IP` → `X-Forwarded-For` 首个公网 IP。
fn auto_forwarded_client_ip(headers: &HeaderMap) -> Option<String> {
if let Some(cf_ip) = header_string(headers, CF_CONNECTING_IP_HEADER) {
@@ -212,12 +151,6 @@ fn forwarded_for_public_ip(headers: &HeaderMap) -> Option<String> {
.map(ToString::to_string)
}
fn trusted_forwarded_client_ip(headers: &HeaderMap) -> Option<String> {
header_string(headers, CF_CONNECTING_IP_HEADER)
.or_else(|| first_forwarded_for_value(headers))
.or_else(|| header_string(headers, REAL_IP_HEADER))
}
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
@@ -236,19 +169,3 @@ fn first_forwarded_for_value(headers: &HeaderMap) -> Option<String> {
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
fn parse_trusted_proxy_entry(entry: &str) -> Result<IpNet, TrustedProxyConfigError> {
if let Ok(network) = entry.parse::<IpNet>() {
return Ok(network);
}
entry
.parse::<IpAddr>()
.map(IpNet::from)
.map_err(|_| invalid_trusted_proxy_entry(entry))
}
fn invalid_trusted_proxy_entry(entry: &str) -> TrustedProxyConfigError {
TrustedProxyConfigError::InvalidEntry {
entry: entry.to_string(),
}
}
@@ -10,9 +10,9 @@ use tower_http::{
classify::{ServerErrorsAsFailures, ServerErrorsFailureClass, SharedClassifier},
trace::TraceLayer,
};
use tracing::{info, warn, Span};
use tracing::{debug, info, warn, Span};
use crate::http::middleware::request_id::RequestId;
use crate::api::middleware::request_id::{ClientIp, RequestId};
/// HTTP tracing layer type.
pub type HttpTraceLayer = TraceLayer<
@@ -36,27 +36,25 @@ pub fn http_trace_layer() -> HttpTraceLayer {
fn make_http_span(request: &Request<Body>) -> Span {
let request_id = request_id(request);
let client_ip = client_ip(request);
let uri = sanitized_uri(request);
tracing::info_span!(
"http",
request_id = request_id.as_deref(),
client_ip,
method = %request.method(),
uri = %uri
)
}
fn on_http_request(request: &Request<Body>, _span: &Span) {
let request_id = request_id(request);
let uri = sanitized_uri(request);
info!(
request_id = request_id.as_deref(),
method = %request.method(),
uri = %uri,
"received HTTP request"
);
fn on_http_request(_request: &Request<Body>, _span: &Span) {
debug!("received HTTP request");
}
fn on_http_response(response: &Response<Body>, latency: Duration, _span: &Span) {
if response.status().is_server_error() {
return;
}
info!(
status = response.status().as_u16(),
latency_ms = latency.as_millis(),
@@ -83,6 +81,10 @@ fn request_id(request: &Request<Body>) -> Option<String> {
.map(|rid| rid.as_str().to_string())
}
fn client_ip(request: &Request<Body>) -> Option<&str> {
request.extensions().get::<ClientIp>().map(ClientIp::as_str)
}
fn sanitized_uri(request: &Request<Body>) -> String {
let uri = request.uri();
let path = uri.path();
+9
View File
@@ -0,0 +1,9 @@
//! 入站 HTTP API 与静态资源。
pub mod admin;
pub mod assets;
pub mod client;
pub mod middleware;
pub mod router;
pub use router::AppState;
+101
View File
@@ -0,0 +1,101 @@
//! 顶层 HTTP 路由 —— 组合 OpenAI API、管理端 API 和静态资源服务。
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use async_trait::async_trait;
use axum::{extract::State, http::StatusCode, middleware, routing::get, Router};
use crate::{
api::{
admin, assets, client,
middleware::{request_id::attach_request_id, trace::http_trace_layer},
},
auth::service::SessionService,
dispatch::{affinity::SessionAffinityService, service::ResponseDispatchService},
fleet::{
manage::AccountManageService, pool::AccountPoolService, refresh::TokenRefreshService,
store::AccountStore,
},
keys::{manage::KeyManageService, service::KeyVerifier},
models::service::ModelService,
settings::service::SettingsService,
telemetry::{
account_usage::query::AccountUsageQueryService, ops::query::OpsQueryService,
usage::query::UsageQueryService,
},
upstream::openai::{fingerprint::RuntimeFingerprint, token_client::OpenAiTokenClient},
};
/// HTTP API 所需的领域服务集合。
#[derive(Clone)]
pub struct ApiServices {
pub health_probe: Arc<dyn HealthProbe>,
pub models: Arc<ModelService>,
pub accounts: Arc<dyn AccountStore>,
pub client_keys: Arc<KeyVerifier>,
pub admin_client_keys: Arc<KeyManageService>,
pub admin_sessions: Arc<SessionService>,
pub settings: Arc<SettingsService>,
pub admin_accounts: Arc<AccountManageService>,
pub usage_records: Arc<UsageQueryService>,
pub ops_errors: Arc<OpsQueryService>,
pub usage: Arc<AccountUsageQueryService>,
pub account_pool: Arc<AccountPoolService>,
pub token_refresh: Arc<TokenRefreshService<OpenAiTokenClient>>,
pub responses: Arc<ResponseDispatchService>,
pub session_affinity: Arc<SessionAffinityService>,
pub fingerprint: RuntimeFingerprint,
pub process_control: Arc<dyn ProcessControl>,
}
/// 由 bootstrap 实现的进程生命周期控制端口。
pub trait ProcessControl: Send + Sync + 'static {
fn request_shutdown(&self);
fn request_restart(&self, executable_path: PathBuf);
fn subscribe_shutdown(&self) -> tokio::sync::broadcast::Receiver<()>;
}
/// 由 bootstrap 实现的基础设施健康检查端口。
#[async_trait]
pub trait HealthProbe: Send + Sync + 'static {
async fn check(&self) -> Result<(), String>;
}
/// HTTP handler 共享状态。
#[derive(Clone)]
pub struct AppState {
pub services: ApiServices,
}
/// 默认前端构建产物目录。
pub const DEFAULT_ASSET_DIST_DIR: &str = "web/dist";
/// 构造整个 HTTP 服务路由。
pub fn router() -> Router<AppState> {
router_with_assets(DEFAULT_ASSET_DIST_DIR)
}
/// 使用指定前端构建产物目录构造整个 HTTP 服务路由。
pub fn router_with_assets(dist_dir: impl AsRef<Path>) -> Router<AppState> {
let traced_routes = Router::new()
.merge(client::router::router())
.merge(admin::router::router())
.fallback_service(assets::spa_router(dist_dir))
.layer(http_trace_layer());
Router::new()
.route("/healthz", get(healthz))
.merge(traced_routes)
.layer(middleware::from_fn(attach_request_id))
}
async fn healthz(State(state): State<AppState>) -> StatusCode {
if let Err(error) = state.services.health_probe.check().await {
tracing::warn!(error, "health check failed");
return StatusCode::SERVICE_UNAVAILABLE;
}
StatusCode::NO_CONTENT
}
+5
View File
@@ -0,0 +1,5 @@
//! 管理员认证与 Redis 会话。
pub mod service;
pub mod store;
pub mod types;
+195
View File
@@ -0,0 +1,195 @@
//! 管理员认证领域服务。
use std::{collections::HashMap, sync::Arc};
use chrono::{DateTime, Duration, Utc};
use tokio::sync::Mutex;
use crate::infra::identity::{
generate_admin_session_token, hash_admin_password, verify_admin_password,
};
use super::{
store::{PgAdminUserStore, RedisAdminSessionStore},
types::{LoginSession, SessionError},
};
const LOGIN_FAILURE_WINDOW: Duration = Duration::minutes(15);
const LOGIN_LOCK_DURATION: Duration = Duration::minutes(15);
const LOGIN_MAX_FAILURES: u32 = 5;
/// 管理员会话服务。
#[derive(Clone)]
pub struct SessionService {
users: PgAdminUserStore,
sessions: RedisAdminSessionStore,
default_username: String,
session_ttl_minutes: u64,
login_failures: Arc<Mutex<HashMap<String, LoginFailureState>>>,
}
#[derive(Debug, Clone)]
struct LoginFailureState {
failures: u32,
first_failed_at: DateTime<Utc>,
locked_until: Option<DateTime<Utc>>,
}
impl SessionService {
/// 构造服务。
pub fn new(
users: PgAdminUserStore,
sessions: RedisAdminSessionStore,
default_username: String,
session_ttl_minutes: u64,
) -> Self {
Self {
users,
sessions,
default_username,
session_ttl_minutes,
login_failures: Arc::new(Mutex::new(HashMap::new())),
}
}
/// 校验管理员会话。
pub async fn validate(&self, session_id: Option<&str>) -> Result<bool, SessionError> {
let Some(session_id) = session_id else {
return Ok(false);
};
self.sessions
.validate_session(session_id)
.await
.map_err(|_| SessionError::Validate)
}
/// 确保默认管理员存在。
pub async fn ensure_default_admin(&self, password: &str) -> Result<bool, SessionError> {
let password_hash =
hash_admin_password(password).map_err(|_| SessionError::HashPassword)?;
self.users
.ensure_default_admin(&password_hash)
.await
.map_err(|_| SessionError::CreateAdmin)
}
/// 登录并创建管理员会话。
pub async fn login(
&self,
source: &str,
username: Option<&str>,
password: &str,
) -> Result<Option<LoginSession>, SessionError> {
let source = login_source_key(source);
self.ensure_login_allowed(&source).await?;
let username = username.unwrap_or(&self.default_username);
if username != self.default_username.as_str() {
self.record_failed_login(&source).await;
return Ok(None);
}
let Some(admin) = self
.users
.load_first_admin()
.await
.map_err(|_| SessionError::LoadAdmin)?
else {
self.record_failed_login(&source).await;
return Ok(None);
};
let password_matches = verify_admin_password(password, &admin.password_hash)
.map_err(|_| SessionError::VerifyPassword)?;
if !password_matches {
self.record_failed_login(&source).await;
return Ok(None);
}
self.clear_failed_login(&source).await;
let session_id = generate_admin_session_token();
let ttl_minutes = self.session_ttl_minutes.min(i64::MAX as u64) as i64;
let expires_at = Utc::now() + Duration::minutes(ttl_minutes);
self.sessions
.create_session(
&session_id,
&admin.id,
Duration::minutes(ttl_minutes.max(1)),
)
.await
.map_err(|_| SessionError::CreateSession)?;
Ok(Some(LoginSession {
session_id,
expires_at,
}))
}
/// 删除管理员会话。
pub async fn delete_session(&self, session_id: &str) -> Result<bool, SessionError> {
self.sessions
.delete_session(session_id)
.await
.map_err(|_| SessionError::DeleteSession)
}
async fn ensure_login_allowed(&self, source: &str) -> Result<(), SessionError> {
let now = Utc::now();
let login_locked = {
let mut failures = self.login_failures.lock().await;
retain_active_login_failures(&mut failures, now);
failures
.get(source)
.and_then(|state| state.locked_until)
.is_some_and(|locked_until| locked_until > now)
};
if login_locked {
return Err(SessionError::LoginThrottled);
}
Ok(())
}
async fn record_failed_login(&self, source: &str) {
let now = Utc::now();
let mut failures = self.login_failures.lock().await;
retain_active_login_failures(&mut failures, now);
let state = failures
.entry(source.to_string())
.or_insert_with(|| LoginFailureState {
failures: 0,
first_failed_at: now,
locked_until: None,
});
if now - state.first_failed_at > LOGIN_FAILURE_WINDOW {
state.failures = 0;
state.first_failed_at = now;
state.locked_until = None;
}
state.failures += 1;
if state.failures >= LOGIN_MAX_FAILURES {
state.locked_until = Some(now + LOGIN_LOCK_DURATION);
}
drop(failures);
}
async fn clear_failed_login(&self, source: &str) {
self.login_failures.lock().await.remove(source);
}
}
fn login_source_key(source: &str) -> String {
let source = source.trim();
if source.is_empty() {
"unknown".to_string()
} else {
source.to_string()
}
}
fn retain_active_login_failures(
failures: &mut HashMap<String, LoginFailureState>,
now: DateTime<Utc>,
) {
failures.retain(|_, state| {
state
.locked_until
.is_some_and(|locked_until| locked_until > now)
|| now - state.first_failed_at <= LOGIN_FAILURE_WINDOW
});
}
+137
View File
@@ -0,0 +1,137 @@
//! 管理员用户与会话存储。
use chrono::{DateTime, Duration, Utc};
use redis::AsyncCommands;
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use thiserror::Error;
use crate::infra::{identity::hash_credential, redis::RedisConnection};
/// 已持久化的管理员用户。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StoredAdminUser {
/// 管理员用户 ID。
pub id: String,
/// 管理员密码哈希。
pub password_hash: String,
}
/// 管理认证存储错误。
#[derive(Debug, Error)]
pub enum AuthStoreError {
#[error("PostgreSQL admin user operation failed: {0}")]
Postgres(#[from] sqlx::Error),
#[error("Redis admin session operation failed: {0}")]
Redis(#[from] redis::RedisError),
#[error("invalid Redis admin session value: {0}")]
Json(#[from] serde_json::Error),
}
pub type AuthStoreResult<T> = Result<T, AuthStoreError>;
/// PostgreSQL 管理员用户存储。
#[derive(Clone)]
pub struct PgAdminUserStore {
pool: PgPool,
}
impl PgAdminUserStore {
/// 构造存储。
pub fn new(pool: PgPool) -> Self {
Self { pool }
}
/// 如果还没有管理员用户,则创建默认管理员。
pub async fn ensure_default_admin(&self, password_hash: &str) -> AuthStoreResult<bool> {
let now = Utc::now();
let result = sqlx::query(
r"
insert into admin_users (id, password_hash, created_at, updated_at)
select $1, $2, $3, $3
where not exists (select 1 from admin_users)
on conflict do nothing",
)
.bind("admin_1")
.bind(password_hash)
.bind(now)
.execute(&self.pool)
.await?;
Ok(result.rows_affected() > 0)
}
/// 读取第一个管理员用户。
pub async fn load_first_admin(&self) -> AuthStoreResult<Option<StoredAdminUser>> {
let row = sqlx::query_as::<_, (String, String)>(
"select id, password_hash from admin_users order by created_at asc, id asc limit 1",
)
.fetch_optional(&self.pool)
.await?;
Ok(row.map(|(id, password_hash)| StoredAdminUser { id, password_hash }))
}
}
/// Redis 管理员会话存储。
#[derive(Clone)]
pub struct RedisAdminSessionStore {
redis: RedisConnection,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct StoredAdminSession {
user_id: String,
created_at: DateTime<Utc>,
}
impl RedisAdminSessionStore {
pub fn new(redis: RedisConnection) -> Self {
Self { redis }
}
pub async fn create_session(
&self,
session_id: &str,
user_id: &str,
ttl: Duration,
) -> AuthStoreResult<()> {
let key = self.session_key(session_id);
let value = serde_json::to_string(&StoredAdminSession {
user_id: user_id.to_string(),
created_at: Utc::now(),
})?;
let ttl_seconds = ttl.num_seconds().max(1) as u64;
let mut connection = self.redis.manager();
let _: () = redis::cmd("SET")
.arg(key)
.arg(value)
.arg("EX")
.arg(ttl_seconds)
.query_async(&mut connection)
.await?;
Ok(())
}
pub async fn validate_session(&self, session_id: &str) -> AuthStoreResult<bool> {
let mut connection = self.redis.manager();
let value: Option<String> = connection.get(self.session_key(session_id)).await?;
let Some(value) = value else {
return Ok(false);
};
let session: StoredAdminSession = serde_json::from_str(&value)?;
Ok(!session.user_id.trim().is_empty())
}
pub async fn delete_session(&self, session_id: &str) -> AuthStoreResult<bool> {
let mut connection = self.redis.manager();
let deleted: u64 = connection.del(self.session_key(session_id)).await?;
Ok(deleted > 0)
}
fn session_key(&self, session_id: &str) -> String {
self.redis
.key(&format!("admin:session:{}", hash_credential(session_id)))
}
}
+42
View File
@@ -0,0 +1,42 @@
//! 管理员认证领域类型。
use chrono::{DateTime, Utc};
use thiserror::Error;
/// 管理员登录成功后的会话。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LoginSession {
/// 会话 ID。
pub session_id: String,
/// 过期时间。
pub expires_at: DateTime<Utc>,
}
/// 管理员会话错误。
#[derive(Debug, Error)]
pub enum SessionError {
/// 校验管理员会话失败。
#[error("failed to validate admin session")]
Validate,
/// 计算管理员密码哈希失败。
#[error("failed to hash admin password")]
HashPassword,
/// 创建默认管理员失败。
#[error("failed to create default admin user")]
CreateAdmin,
/// 读取管理员用户失败。
#[error("failed to load admin user")]
LoadAdmin,
/// 校验管理员密码失败。
#[error("failed to verify admin password")]
VerifyPassword,
/// 创建管理员会话失败。
#[error("failed to create admin session")]
CreateSession,
/// 删除管理员会话失败。
#[error("failed to delete admin session")]
DeleteSession,
/// 管理员登录尝试过于频繁。
#[error("too many admin login attempts")]
LoginThrottled,
}
@@ -1,6 +1,8 @@
use std::collections::BTreeMap;
use std::{collections::BTreeMap, env, path::Path};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use serde::{Deserialize, Serialize};
use url::Url;
const WEAK_ADMIN_PASSWORDS: &[&str] = &[
"",
@@ -24,9 +26,6 @@ pub struct AppConfig {
/// 运行时模型别名,由数据库设置加载,配置文件不承载该字段。
#[serde(skip)]
pub model_aliases: BTreeMap<String, String>,
/// 运行时模型账号路由,由 `model_account_routes` 表加载,配置文件不承载该字段。
#[serde(skip)]
pub model_account_routes: BTreeMap<String, Vec<String>>,
/// 认证与刷新配置,由固定默认值和数据库运行时设置承载,配置文件不承载该字段。
#[serde(skip)]
pub auth: AuthConfig,
@@ -35,12 +34,14 @@ pub struct AppConfig {
pub quota: QuotaConfig,
/// 数据库配置。
pub database: DatabaseConfig,
/// Redis 运行态存储配置。
pub redis: RedisConfig,
/// TLS 偏好配置。
#[serde(default)]
pub tls: TlsConfig,
/// WebSocket 连接池配置。
/// WebSocket 连接池启动设置。
#[serde(default)]
pub ws_pool: WebSocketPoolConfig,
pub ws_pool: WebSocketPoolSettings,
/// 上游请求指纹默认配置。
#[serde(default)]
pub fingerprint: FingerprintConfig,
@@ -50,6 +51,9 @@ pub struct AppConfig {
/// 日志配置。
#[serde(default)]
pub logging: LoggingConfig,
/// PostgreSQL 遥测事实记录配置。
#[serde(default)]
pub telemetry: TelemetryConfig,
}
/// HTTP 监听配置。
@@ -60,9 +64,6 @@ pub struct ServerConfig {
pub host: String,
/// 监听端口。
pub port: u16,
/// 可信反向代理 IP 或 CIDR。只有这些 peer 的转发 IP 头会被采信。
#[serde(default)]
pub trusted_proxies: Vec<String>,
}
/// 上游 API 地址配置。
@@ -98,12 +99,12 @@ pub struct AuthConfig {
impl Default for AuthConfig {
fn default() -> Self {
Self {
refresh_margin_seconds: 300,
refresh_margin_seconds: 3600,
refresh_enabled: true,
refresh_concurrency: 2,
max_concurrent_per_account: 3,
request_interval_ms: 50,
rotation_strategy: "least_used".to_string(),
rotation_strategy: "smart".to_string(),
tier_priority: Vec::new(),
oauth_client_id: "app_EMoamEEZ73f0CkXaXp7hrann".to_string(),
oauth_token_endpoint: "https://auth.openai.com/oauth/token".to_string(),
@@ -143,7 +144,15 @@ fn default_skip_exhausted() -> bool {
/// 数据库连接配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
pub struct DatabaseConfig {
/// SQLite 连接 URL。
/// PostgreSQL 连接 URL。
pub url: String,
}
/// Redis 连接配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct RedisConfig {
/// Redis 连接 URL。
pub url: String,
}
@@ -154,24 +163,36 @@ pub struct TlsConfig {
pub force_http11: bool,
}
/// WebSocket 连接池配置。
/// WebSocket 连接池启动设置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct WebSocketPoolConfig {
pub struct WebSocketPoolSettings {
/// 是否启用连接池。
pub enabled: bool,
/// 连接最大存活时长。
pub max_age_ms: u64,
/// 单账号最大连接数。
pub max_per_account: usize,
/// 首个真实输出(首事件)到达前的绝对超时;`0` 表示禁用。
///
/// 从发出 `response.create` 起算,覆盖建连/发送/上游排队/思考的全程,直到收到
/// 首个真实内容帧(`response.created`/`response.in_progress` 等纯生命周期帧不计)。
/// 超时判定为连接落到病态上游后端,丢弃并换新连接重试。
#[serde(default = "default_ws_initial_event_timeout_ms")]
pub initial_event_timeout_ms: u64,
}
impl Default for WebSocketPoolConfig {
fn default_ws_initial_event_timeout_ms() -> u64 {
20_000
}
impl Default for WebSocketPoolSettings {
fn default() -> Self {
Self {
enabled: true,
max_age_ms: 3_300_000,
max_per_account: 8,
initial_event_timeout_ms: default_ws_initial_event_timeout_ms(),
}
}
}
@@ -266,12 +287,10 @@ impl FingerprintHeaderConfig {
/// 管理员初始化与会话配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct AdminConfig {
/// 会话有效期(分钟)。
pub session_ttl_minutes: u64,
/// 会话清理周期(秒)。
#[serde(default = "default_session_cleanup_interval")]
pub session_cleanup_interval_secs: u64,
/// 默认管理员用户名(首次启动时创建)
#[serde(default = "default_admin_username")]
pub default_username: String,
@@ -284,7 +303,6 @@ impl Default for AdminConfig {
fn default() -> Self {
Self {
session_ttl_minutes: 1440,
session_cleanup_interval_secs: default_session_cleanup_interval(),
default_username: default_admin_username(),
default_password: default_admin_password(),
}
@@ -315,10 +333,6 @@ pub enum AdminConfigError {
WeakDefaultPassword,
}
fn default_session_cleanup_interval() -> u64 {
3600 // 默认每小时清理一次过期会话
}
fn default_admin_username() -> String {
"admin".to_string()
}
@@ -327,24 +341,142 @@ fn default_admin_password() -> String {
String::new()
}
/// 日志持久化配置。
/// 应用结构化日志配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct LoggingConfig {
/// 日志目录。
pub directory: String,
/// 保留天数。
pub retention_days: usize,
/// 是否启用日志。
pub enabled: bool,
/// 默认日志级别;`RUST_LOG` 存在时优先使用环境变量。
pub level: String,
/// 是否写入标准输出。
pub stdout: bool,
/// 文件日志配置。
pub file: FileLoggingConfig,
}
impl Default for LoggingConfig {
fn default() -> Self {
Self {
directory: ".runtime/logs".to_string(),
retention_days: 14,
enabled: true,
level: "info".to_string(),
stdout: true,
file: FileLoggingConfig::default(),
}
}
}
/// 应用结构化文件日志配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct FileLoggingConfig {
/// 是否写入轮转文件。
pub enabled: bool,
/// 日志目录。
pub directory: String,
/// 按中国自然日计算的保留天数。
pub retention_days: usize,
/// 单文件大小上限,单位 MiB。
pub max_file_size_mb: u64,
/// 文件总数上限。
pub max_files: usize,
}
impl Default for FileLoggingConfig {
fn default() -> Self {
Self {
enabled: true,
directory: ".runtime/logs".to_string(),
retention_days: 14,
max_file_size_mb: 20,
max_files: 20,
}
}
}
/// PostgreSQL 遥测事实记录配置。
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct TelemetryConfig {
/// 是否记录成功与失败代理事实。
pub enabled: bool,
}
impl Default for TelemetryConfig {
fn default() -> Self {
Self { enabled: true }
}
}
const CONFIG_FILE_ENV: &str = "CPR_CONFIG_FILE";
const ADMIN_DEFAULT_PASSWORD_ENV: &str = "CPR_ADMIN_DEFAULT_PASSWORD";
const POSTGRES_PASSWORD_ENV: &str = "CPR_POSTGRES_PASSWORD";
const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD";
impl AppConfig {
/// 从 `CPR_CONFIG_FILE` 或当前目录 `config.yaml` 加载配置文件,并应用运行时密码。
pub fn load() -> Result<Self, ::config::ConfigError> {
let config_file = env::var(CONFIG_FILE_ENV)
.ok()
.filter(|value| !value.trim().is_empty());
match config_file {
Some(path) => load_file(path),
None => Self::load_from_dir("."),
}
}
/// 从指定目录加载配置文件,并应用运行时密码。
pub fn load_from_dir(config_dir: impl AsRef<Path>) -> Result<Self, ::config::ConfigError> {
load_file(config_dir.as_ref().join("config.yaml"))
}
}
fn load_file(config_file: impl AsRef<Path>) -> Result<AppConfig, ::config::ConfigError> {
let mut app_config: AppConfig = ::config::Config::builder()
.add_source(::config::File::from(config_file.as_ref()).required(true))
.build()?
.try_deserialize()?;
apply_password_override(
&mut app_config.database.url,
"database.url",
POSTGRES_PASSWORD_ENV,
)?;
apply_password_override(&mut app_config.redis.url, "redis.url", REDIS_PASSWORD_ENV)?;
if let Some(password) = password_override(ADMIN_DEFAULT_PASSWORD_ENV)? {
app_config.admin.default_password = password;
}
Ok(app_config)
}
fn apply_password_override(
raw_url: &mut String,
config_key: &str,
env_name: &str,
) -> Result<(), ::config::ConfigError> {
let Some(password) = password_override(env_name)? else {
return Ok(());
};
let mut url = Url::parse(raw_url).map_err(|_| {
::config::ConfigError::Message(format!(
"{config_key} must be a valid URL when {env_name} is set"
))
})?;
let encoded_password = utf8_percent_encode(&password, NON_ALPHANUMERIC).to_string();
url.set_password(Some(&encoded_password)).map_err(|()| {
::config::ConfigError::Message(format!(
"{config_key} does not support a password override from {env_name}"
))
})?;
*raw_url = url.into();
Ok(())
}
fn password_override(env_name: &str) -> Result<Option<String>, ::config::ConfigError> {
match env::var(env_name) {
Ok(password) if password.is_empty() => Err(::config::ConfigError::Message(format!(
"{env_name} must not be empty"
))),
Ok(password) => Ok(Some(password)),
Err(env::VarError::NotPresent) => Ok(None),
Err(env::VarError::NotUnicode(_)) => Err(::config::ConfigError::Message(format!(
"{env_name} must contain valid UTF-8"
))),
}
}
+460
View File
@@ -0,0 +1,460 @@
use futures::TryStreamExt;
use sqlx::{types::Json, Postgres, Row, SqlitePool, Transaction};
use crate::infra::identity::hash_credential;
use super::{
parse_cookie_expiration, parse_json, parse_optional_json, parse_optional_timestamp,
parse_timestamp, ImportSqliteError, ImportSqliteReport,
};
pub(super) async fn import_core_tables(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
import_admin_users(source, target, report).await?;
import_client_api_keys(source, target, report).await?;
import_runtime_settings(source, target, report).await?;
import_accounts(source, target, report).await?;
import_account_usage(source, target, report).await?;
import_account_cookies(source, target, report).await?;
import_fingerprints(source, target, report).await?;
import_fingerprint_history(source, target, report).await
}
async fn import_admin_users(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, password_hash, created_at, updated_at from admin_users order by id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
sqlx::query(
"insert into admin_users (id, password_hash, created_at, updated_at)
values ($1, $2, $3, $4)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<String, _>("password_hash")?)
.bind(parse_timestamp(
"admin_users",
"created_at",
row.try_get("created_at")?,
)?)
.bind(parse_timestamp(
"admin_users",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("admin_users");
}
Ok(())
}
async fn import_client_api_keys(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, name, prefix, key, label, enabled, created_at, last_used_at
from client_api_keys order by id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
sqlx::query(
"insert into client_api_keys
(id, name, prefix, key, label, enabled, created_at, last_used_at)
values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<String, _>("name")?)
.bind(row.try_get::<String, _>("prefix")?)
.bind(row.try_get::<String, _>("key")?)
.bind(row.try_get::<Option<String>, _>("label")?)
.bind(row.try_get::<i64, _>("enabled")? == 1)
.bind(parse_timestamp(
"client_api_keys",
"created_at",
row.try_get("created_at")?,
)?)
.bind(parse_optional_timestamp(
"client_api_keys",
"last_used_at",
row.try_get("last_used_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("client_api_keys");
}
Ok(())
}
async fn import_runtime_settings(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, model_aliases_json, refresh_margin_seconds, refresh_concurrency,
max_concurrent_per_account, request_interval_ms, rotation_strategy,
admin_api_key, updated_at
from runtime_settings order by id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
let aliases: String = row.try_get("model_aliases_json")?;
let admin_key = row
.try_get::<Option<String>, _>("admin_api_key")?
.and_then(|key| (!key.trim().is_empty()).then(|| hash_credential(&key)));
sqlx::query(
"insert into runtime_settings
(id, model_aliases_json, refresh_margin_seconds, refresh_concurrency,
max_concurrent_per_account, request_interval_ms, rotation_strategy,
admin_api_key_hash, usage_retention_days, ops_error_retention_days,
bucket_retention_days, updated_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, 30, 30, 90, $9)",
)
.bind(row.try_get::<i64, _>("id")?)
.bind(Json(parse_json(
"runtime_settings",
"model_aliases_json",
&aliases,
)?))
.bind(row.try_get::<i64, _>("refresh_margin_seconds")?)
.bind(row.try_get::<i64, _>("refresh_concurrency")?)
.bind(row.try_get::<i64, _>("max_concurrent_per_account")?)
.bind(row.try_get::<i64, _>("request_interval_ms")?)
.bind(row.try_get::<String, _>("rotation_strategy")?)
.bind(admin_key)
.bind(parse_timestamp(
"runtime_settings",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("runtime_settings");
}
Ok(())
}
async fn import_accounts(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, email, chatgpt_account_id, chatgpt_user_id, label, plan_type,
access_token, refresh_token, access_token_expires_at, next_refresh_at,
status, quota_json, quota_fetched_at, quota_limit_reached,
quota_verify_required, quota_cooldown_until, cloudflare_cooldown_until,
added_at, updated_at
from accounts order by id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
let source_status = row.try_get::<String, _>("status")?;
let (status, next_refresh_at) = if source_status == "refreshing" {
report.add_normalized("accounts.refreshing_to_expired");
("expired", None)
} else {
(
source_status.as_str(),
parse_optional_timestamp(
"accounts",
"next_refresh_at",
row.try_get("next_refresh_at")?,
)?,
)
};
let quota = parse_optional_json(
"accounts",
"quota_json",
row.try_get::<Option<String>, _>("quota_json")?,
)?
.map(Json);
sqlx::query(
"insert into accounts
(id, email, chatgpt_account_id, chatgpt_user_id, label, plan_type,
access_token, refresh_token, access_token_expires_at, next_refresh_at,
status, quota_json, quota_fetched_at, quota_limit_reached,
quota_verify_required, quota_cooldown_until, cloudflare_cooldown_until,
added_at, updated_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
$14, $15, $16, $17, $18, $19)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<Option<String>, _>("email")?)
.bind(row.try_get::<Option<String>, _>("chatgpt_account_id")?)
.bind(row.try_get::<Option<String>, _>("chatgpt_user_id")?)
.bind(row.try_get::<Option<String>, _>("label")?)
.bind(row.try_get::<Option<String>, _>("plan_type")?)
.bind(row.try_get::<String, _>("access_token")?)
.bind(row.try_get::<Option<String>, _>("refresh_token")?)
.bind(parse_optional_timestamp(
"accounts",
"access_token_expires_at",
row.try_get("access_token_expires_at")?,
)?)
.bind(next_refresh_at)
.bind(status)
.bind(quota)
.bind(parse_optional_timestamp(
"accounts",
"quota_fetched_at",
row.try_get("quota_fetched_at")?,
)?)
.bind(row.try_get::<i64, _>("quota_limit_reached")? == 1)
.bind(row.try_get::<i64, _>("quota_verify_required")? == 1)
.bind(parse_optional_timestamp(
"accounts",
"quota_cooldown_until",
row.try_get("quota_cooldown_until")?,
)?)
.bind(parse_optional_timestamp(
"accounts",
"cloudflare_cooldown_until",
row.try_get("cloudflare_cooldown_until")?,
)?)
.bind(parse_timestamp(
"accounts",
"added_at",
row.try_get("added_at")?,
)?)
.bind(parse_timestamp(
"accounts",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("accounts");
}
Ok(())
}
async fn import_account_usage(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query("select * from account_usage order by account_id").fetch(source);
while let Some(row) = rows.try_next().await? {
sqlx::query(
"insert into account_usage
(account_id, request_count, empty_response_count, input_tokens, output_tokens,
cached_tokens, reasoning_tokens, total_tokens, image_input_tokens,
image_output_tokens, image_request_count, image_request_failed_count,
window_request_count, window_input_tokens, window_output_tokens,
window_cached_tokens, window_image_input_tokens, window_image_output_tokens,
window_image_request_count, window_image_request_failed_count,
window_started_at, window_reset_at, limit_window_seconds, last_used_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
$13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)",
)
.bind(row.try_get::<String, _>("account_id")?)
.bind(row.try_get::<i64, _>("request_count")?)
.bind(row.try_get::<i64, _>("empty_response_count")?)
.bind(row.try_get::<i64, _>("input_tokens")?)
.bind(row.try_get::<i64, _>("output_tokens")?)
.bind(row.try_get::<i64, _>("cached_tokens")?)
.bind(row.try_get::<i64, _>("reasoning_tokens")?)
.bind(row.try_get::<i64, _>("total_tokens")?)
.bind(row.try_get::<i64, _>("image_input_tokens")?)
.bind(row.try_get::<i64, _>("image_output_tokens")?)
.bind(row.try_get::<i64, _>("image_request_count")?)
.bind(row.try_get::<i64, _>("image_request_failed_count")?)
.bind(row.try_get::<i64, _>("window_request_count")?)
.bind(row.try_get::<i64, _>("window_input_tokens")?)
.bind(row.try_get::<i64, _>("window_output_tokens")?)
.bind(row.try_get::<i64, _>("window_cached_tokens")?)
.bind(row.try_get::<i64, _>("window_image_input_tokens")?)
.bind(row.try_get::<i64, _>("window_image_output_tokens")?)
.bind(row.try_get::<i64, _>("window_image_request_count")?)
.bind(row.try_get::<i64, _>("window_image_request_failed_count")?)
.bind(parse_optional_timestamp(
"account_usage",
"window_started_at",
row.try_get("window_started_at")?,
)?)
.bind(parse_optional_timestamp(
"account_usage",
"window_reset_at",
row.try_get("window_reset_at")?,
)?)
.bind(row.try_get::<Option<i64>, _>("limit_window_seconds")?)
.bind(parse_optional_timestamp(
"account_usage",
"last_used_at",
row.try_get("last_used_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("account_usage");
}
Ok(())
}
async fn import_account_cookies(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, account_id, domain, name, value, path, expires_at, updated_at
from account_cookies order by id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
let expires_at = row
.try_get::<Option<String>, _>("expires_at")?
.and_then(|value| match parse_cookie_expiration(&value) {
Some(timestamp) => Some(timestamp),
None => {
report.add_cookie_expiration_parse_failure();
None
}
});
sqlx::query(
"insert into account_cookies
(id, account_id, domain, name, value, path, expires_at, updated_at)
values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<String, _>("account_id")?)
.bind(row.try_get::<String, _>("domain")?)
.bind(row.try_get::<String, _>("name")?)
.bind(row.try_get::<String, _>("value")?)
.bind(row.try_get::<String, _>("path")?)
.bind(expires_at)
.bind(parse_timestamp(
"account_cookies",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("account_cookies");
}
Ok(())
}
async fn import_fingerprints(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query("select * from fingerprints order by id").fetch(source);
while let Some(row) = rows.try_next().await? {
let default_headers: String = row.try_get("default_headers_json")?;
let header_order: String = row.try_get("header_order_json")?;
sqlx::query(
"insert into fingerprints
(id, originator, app_version, build_number, platform, arch, chromium_version,
user_agent_template, default_headers_json, header_order_json, source,
created_at, updated_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<String, _>("originator")?)
.bind(row.try_get::<String, _>("app_version")?)
.bind(row.try_get::<String, _>("build_number")?)
.bind(row.try_get::<String, _>("platform")?)
.bind(row.try_get::<String, _>("arch")?)
.bind(row.try_get::<String, _>("chromium_version")?)
.bind(row.try_get::<String, _>("user_agent_template")?)
.bind(Json(parse_json(
"fingerprints",
"default_headers_json",
&default_headers,
)?))
.bind(Json(parse_json(
"fingerprints",
"header_order_json",
&header_order,
)?))
.bind(row.try_get::<String, _>("source")?)
.bind(parse_timestamp(
"fingerprints",
"created_at",
row.try_get("created_at")?,
)?)
.bind(parse_timestamp(
"fingerprints",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("fingerprints");
}
Ok(())
}
async fn import_fingerprint_history(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows =
sqlx::query("select * from fingerprint_update_history order by id").fetch(source);
while let Some(row) = rows.try_next().await? {
let manifest = parse_optional_json(
"fingerprint_update_history",
"manifest_json",
row.try_get::<Option<String>, _>("manifest_json")?,
)?
.map(Json);
sqlx::query(
"insert into fingerprint_update_history
(id, current_fingerprint_id, app_version, build_number, chromium_version,
source, manifest_json, created_at)
values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<String, _>("current_fingerprint_id")?)
.bind(row.try_get::<String, _>("app_version")?)
.bind(row.try_get::<String, _>("build_number")?)
.bind(row.try_get::<Option<String>, _>("chromium_version")?)
.bind(row.try_get::<String, _>("source")?)
.bind(manifest)
.bind(parse_timestamp(
"fingerprint_update_history",
"created_at",
row.try_get("created_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("fingerprint_update_history");
}
Ok(())
}
pub(super) async fn count_discarded_runtime_tables(
source: &SqlitePool,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let (sessions, affinities, leases, snapshots, model_usage): (i64, i64, i64, i64, i64) =
sqlx::query_as(
"select
(select count(*) from admin_sessions),
(select count(*) from session_affinities),
(select count(*) from account_refresh_leases),
(select count(*) from model_plan_snapshots),
(select count(*) from account_model_usage)",
)
.fetch_one(source)
.await?;
report.add_discarded("admin_sessions", sessions.max(0) as u64);
report.add_discarded("session_affinities", affinities.max(0) as u64);
report.add_discarded("account_refresh_leases", leases.max(0) as u64);
report.add_discarded("model_plan_snapshots", snapshots.max(0) as u64);
report.add_discarded("account_model_usage", model_usage.max(0) as u64);
Ok(())
}
+232
View File
@@ -0,0 +1,232 @@
//! 一次性 SQLite v3 到 PostgreSQL 终态库导入。
mod core;
mod telemetry;
use std::{collections::BTreeMap, fmt, path::Path};
use chrono::{DateTime, Utc};
use serde_json::Value;
use sqlx::{
sqlite::{SqliteConnectOptions, SqlitePoolOptions},
PgPool, Postgres, SqlitePool, Transaction,
};
use thiserror::Error;
const SQLITE_SCHEMA_VERSION: i64 = 3;
/// SQLite 历史库导入结果。
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ImportSqliteReport {
imported: BTreeMap<&'static str, u64>,
normalized: BTreeMap<&'static str, u64>,
discarded: BTreeMap<&'static str, u64>,
cookie_expiration_parse_failures: u64,
}
impl ImportSqliteReport {
fn add_imported(&mut self, table: &'static str) {
*self.imported.entry(table).or_default() += 1;
}
fn add_discarded(&mut self, reason: &'static str, count: u64) {
if count > 0 {
*self.discarded.entry(reason).or_default() += count;
}
}
fn add_normalized(&mut self, reason: &'static str) {
*self.normalized.entry(reason).or_default() += 1;
}
fn add_cookie_expiration_parse_failure(&mut self) {
self.cookie_expiration_parse_failures += 1;
}
/// 返回指定目标表导入的行数。
pub fn imported_rows(&self, table: &str) -> u64 {
self.imported.get(table).copied().unwrap_or_default()
}
/// 返回指定原因丢弃的行数。
pub fn discarded_rows(&self, reason: &str) -> u64 {
self.discarded.get(reason).copied().unwrap_or_default()
}
/// 返回指定规则规范化的行数。
pub fn normalized_rows(&self, reason: &str) -> u64 {
self.normalized.get(reason).copied().unwrap_or_default()
}
/// 返回无法解析并降级为会话 Cookie 的过期时间数量。
pub fn cookie_expiration_parse_failures(&self) -> u64 {
self.cookie_expiration_parse_failures
}
}
impl fmt::Display for ImportSqliteReport {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(formatter, "SQLite v3 import completed")?;
for (table, count) in &self.imported {
writeln!(formatter, " imported {table}: {count}")?;
}
for (reason, count) in &self.normalized {
writeln!(formatter, " normalized {reason}: {count}")?;
}
for (reason, count) in &self.discarded {
writeln!(formatter, " discarded {reason}: {count}")?;
}
write!(
formatter,
" cookie expiration parse failures: {}",
self.cookie_expiration_parse_failures
)
}
}
/// SQLite 历史库导入错误。
#[derive(Debug, Error)]
pub enum ImportSqliteError {
#[error("database operation failed: {0}")]
Database(#[from] sqlx::Error),
#[error("unsupported SQLite schema version {actual:?}; expected 3")]
UnsupportedSourceVersion { actual: Option<i64> },
#[error("target PostgreSQL database is not empty")]
TargetNotEmpty,
#[error("invalid timestamp in {table}.{column}: {value}")]
InvalidTimestamp {
table: &'static str,
column: &'static str,
value: String,
},
#[error("invalid JSON in {table}.{column}: {source}")]
InvalidJson {
table: &'static str,
column: &'static str,
#[source]
source: serde_json::Error,
},
}
/// 将只读 SQLite v3 历史库一次性导入空的 PostgreSQL 终态库。
pub async fn import_sqlite(
target: &PgPool,
source_path: impl AsRef<Path>,
) -> Result<ImportSqliteReport, ImportSqliteError> {
let options = SqliteConnectOptions::new()
.filename(source_path.as_ref())
.read_only(true)
.create_if_missing(false);
let source = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await?;
validate_source_version(&source).await?;
let mut transaction = target.begin().await?;
reject_nonempty_target(&mut transaction).await?;
let mut report = ImportSqliteReport::default();
core::import_core_tables(&source, &mut transaction, &mut report).await?;
telemetry::import_telemetry_tables(&source, &mut transaction, &mut report).await?;
core::count_discarded_runtime_tables(&source, &mut report).await?;
transaction.commit().await?;
source.close().await;
Ok(report)
}
async fn validate_source_version(source: &SqlitePool) -> Result<(), ImportSqliteError> {
let (actual,): (Option<i64>,) = sqlx::query_as("select max(version) from schema_migrations")
.fetch_one(source)
.await?;
if actual != Some(SQLITE_SCHEMA_VERSION) {
return Err(ImportSqliteError::UnsupportedSourceVersion { actual });
}
Ok(())
}
async fn reject_nonempty_target(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<(), ImportSqliteError> {
let (rows,): (i64,) = sqlx::query_as(
"select
(select count(*) from admin_users)
+ (select count(*) from client_api_keys)
+ (select count(*) from runtime_settings)
+ (select count(*) from accounts)
+ (select count(*) from account_usage)
+ (select count(*) from usage_records)
+ (select count(*) from ops_error_logs)
+ (select count(*) from request_time_buckets)
+ (select count(*) from account_cookies)
+ (select count(*) from fingerprints)
+ (select count(*) from fingerprint_update_history)",
)
.fetch_one(&mut **transaction)
.await?;
if rows != 0 {
return Err(ImportSqliteError::TargetNotEmpty);
}
Ok(())
}
fn parse_timestamp(
table: &'static str,
column: &'static str,
value: String,
) -> Result<DateTime<Utc>, ImportSqliteError> {
DateTime::parse_from_rfc3339(&value)
.map(|timestamp| timestamp.with_timezone(&Utc))
.map_err(|_| ImportSqliteError::InvalidTimestamp {
table,
column,
value,
})
}
fn parse_optional_timestamp(
table: &'static str,
column: &'static str,
value: Option<String>,
) -> Result<Option<DateTime<Utc>>, ImportSqliteError> {
value
.map(|value| parse_timestamp(table, column, value))
.transpose()
}
fn parse_cookie_expiration(value: &str) -> Option<DateTime<Utc>> {
DateTime::parse_from_rfc3339(value)
.or_else(|_| DateTime::parse_from_rfc2822(value))
.ok()
.map(|timestamp| timestamp.with_timezone(&Utc))
}
fn parse_json(
table: &'static str,
column: &'static str,
value: &str,
) -> Result<Value, ImportSqliteError> {
serde_json::from_str(value).map_err(|source| ImportSqliteError::InvalidJson {
table,
column,
source,
})
}
fn parse_optional_json(
table: &'static str,
column: &'static str,
value: Option<String>,
) -> Result<Option<Value>, ImportSqliteError> {
value
.map(|value| parse_json(table, column, &value))
.transpose()
}
fn normalized_dimension(value: Option<String>) -> String {
value
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "__unknown__".to_string())
}
@@ -0,0 +1,330 @@
use futures::TryStreamExt;
use serde_json::Value;
use sqlx::{types::Json, Postgres, Row, SqlitePool, Transaction};
use super::{
normalized_dimension, parse_json, parse_timestamp, ImportSqliteError, ImportSqliteReport,
};
pub(super) async fn import_telemetry_tables(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
import_mixed_usage_records(source, target, report).await?;
import_ops_error_logs(source, target, report).await?;
import_request_time_buckets(source, target, report).await
}
async fn import_mixed_usage_records(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows = sqlx::query(
"select id, request_id, kind, level, account_id, route, model, status_code,
transport, attempt_index, upstream_status_code, failure_class,
response_id, upstream_request_id, latency_ms, message, metadata_json,
created_at
from usage_records order by created_at, id",
)
.fetch(source);
while let Some(row) = rows.try_next().await? {
let level: String = row.try_get("level")?;
let account_id: Option<String> = row.try_get("account_id")?;
let model: Option<String> = row.try_get("model")?;
let status_code: Option<i64> = row.try_get("status_code")?;
let metadata_text: String = row.try_get("metadata_json")?;
let metadata = parse_json("usage_records", "metadata_json", &metadata_text)?;
if level == "error" {
insert_usage_error(target, &row, metadata).await?;
report.add_imported("ops_error_logs");
continue;
}
let valid_account = account_id
.as_deref()
.map(str::trim)
.is_some_and(|value| !value.is_empty());
let valid_model = model
.as_deref()
.map(str::trim)
.is_some_and(|value| !value.is_empty());
let valid_status = status_code.is_some_and(|status| (200..=399).contains(&status));
if !valid_account || !valid_model || !valid_status {
report.add_discarded("usage_record_noise", 1);
continue;
}
insert_success_record(
target,
&row,
account_id.expect("validated account id"),
model.expect("validated model"),
status_code.expect("validated status code"),
metadata,
)
.await?;
report.add_imported("usage_records");
}
Ok(())
}
async fn insert_success_record(
target: &mut Transaction<'_, Postgres>,
row: &sqlx::sqlite::SqliteRow,
account_id: String,
model: String,
status_code: i64,
mut metadata: Value,
) -> Result<(), ImportSqliteError> {
let requested_model = metadata_string(&metadata, &["requestedModel", "requested_model"]);
let upstream_model = metadata_string(&metadata, &["upstreamModel", "upstream_model"]);
let service_tier = metadata_string(&metadata, &["serviceTier", "service_tier"]);
let first_token_ms = metadata_nonnegative(&metadata, &["firstTokenMs", "first_token_ms"]);
let input_tokens = usage_number(&metadata, &["inputTokens", "input_tokens"]);
let output_tokens = usage_number(&metadata, &["outputTokens", "output_tokens"]);
let cached_tokens = usage_number(&metadata, &["cachedTokens", "cached_tokens"]);
let reasoning_tokens = usage_number(&metadata, &["reasoningTokens", "reasoning_tokens"]);
strip_promoted_success_fields(&mut metadata);
sqlx::query(
"insert into usage_records
(id, request_id, client_api_key_id, kind, route, provider, account_id, model,
requested_model, upstream_model, service_tier, status_code, transport,
attempt_index, response_id, upstream_request_id, latency_ms, first_token_ms,
input_tokens, output_tokens, cached_tokens, reasoning_tokens, message,
metadata_json, created_at)
values ($1, $2, null, $3, $4, 'openai', $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<Option<String>, _>("request_id")?)
.bind(row.try_get::<String, _>("kind")?)
.bind(row.try_get::<Option<String>, _>("route")?)
.bind(account_id)
.bind(model)
.bind(requested_model)
.bind(upstream_model)
.bind(service_tier)
.bind(status_code as i32)
.bind(row.try_get::<Option<String>, _>("transport")?)
.bind(row.try_get::<Option<i64>, _>("attempt_index")?)
.bind(row.try_get::<Option<String>, _>("response_id")?)
.bind(row.try_get::<Option<String>, _>("upstream_request_id")?)
.bind(row.try_get::<Option<i64>, _>("latency_ms")?)
.bind(first_token_ms)
.bind(input_tokens)
.bind(output_tokens)
.bind(cached_tokens)
.bind(reasoning_tokens)
.bind(row.try_get::<String, _>("message")?)
.bind(Json(metadata))
.bind(parse_timestamp(
"usage_records",
"created_at",
row.try_get("created_at")?,
)?)
.execute(&mut **target)
.await?;
Ok(())
}
async fn insert_usage_error(
target: &mut Transaction<'_, Postgres>,
row: &sqlx::sqlite::SqliteRow,
metadata: Value,
) -> Result<(), ImportSqliteError> {
sqlx::query(
"insert into ops_error_logs
(id, request_id, client_api_key_id, kind, provider, account_id, route, model,
status_code, client_status_code, upstream_status_code, transport, attempt_index,
failure_class, response_id, upstream_request_id, latency_ms, message,
metadata_json, created_at)
values ($1, $2, null, $3, 'openai', $4, $5, $6, $7, null, $8, $9,
$10, $11, $12, $13, $14, $15, $16, $17)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<Option<String>, _>("request_id")?)
.bind(row.try_get::<String, _>("kind")?)
.bind(row.try_get::<Option<String>, _>("account_id")?)
.bind(row.try_get::<Option<String>, _>("route")?)
.bind(row.try_get::<Option<String>, _>("model")?)
.bind(optional_status(row.try_get("status_code")?))
.bind(optional_status(row.try_get("upstream_status_code")?))
.bind(row.try_get::<Option<String>, _>("transport")?)
.bind(row.try_get::<Option<i64>, _>("attempt_index")?)
.bind(row.try_get::<Option<String>, _>("failure_class")?)
.bind(row.try_get::<Option<String>, _>("response_id")?)
.bind(row.try_get::<Option<String>, _>("upstream_request_id")?)
.bind(row.try_get::<Option<i64>, _>("latency_ms")?)
.bind(row.try_get::<String, _>("message")?)
.bind(Json(metadata))
.bind(parse_timestamp(
"usage_records",
"created_at",
row.try_get("created_at")?,
)?)
.execute(&mut **target)
.await?;
Ok(())
}
async fn import_ops_error_logs(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows =
sqlx::query("select * from ops_error_logs order by created_at, id").fetch(source);
while let Some(row) = rows.try_next().await? {
let metadata_text: String = row.try_get("metadata_json")?;
sqlx::query(
"insert into ops_error_logs
(id, request_id, client_api_key_id, kind, provider, account_id, route, model,
status_code, client_status_code, upstream_status_code, transport, attempt_index,
failure_class, response_id, upstream_request_id, latency_ms, message,
metadata_json, created_at)
values ($1, $2, null, $3, 'openai', $4, $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18)",
)
.bind(row.try_get::<String, _>("id")?)
.bind(row.try_get::<Option<String>, _>("request_id")?)
.bind(row.try_get::<String, _>("kind")?)
.bind(row.try_get::<Option<String>, _>("account_id")?)
.bind(row.try_get::<Option<String>, _>("route")?)
.bind(row.try_get::<Option<String>, _>("model")?)
.bind(optional_status(row.try_get("status_code")?))
.bind(optional_status(row.try_get("client_status_code")?))
.bind(optional_status(row.try_get("upstream_status_code")?))
.bind(row.try_get::<Option<String>, _>("transport")?)
.bind(row.try_get::<Option<i64>, _>("attempt_index")?)
.bind(row.try_get::<Option<String>, _>("failure_class")?)
.bind(row.try_get::<Option<String>, _>("response_id")?)
.bind(row.try_get::<Option<String>, _>("upstream_request_id")?)
.bind(row.try_get::<Option<i64>, _>("latency_ms")?)
.bind(row.try_get::<String, _>("message")?)
.bind(Json(parse_json(
"ops_error_logs",
"metadata_json",
&metadata_text,
)?))
.bind(parse_timestamp(
"ops_error_logs",
"created_at",
row.try_get("created_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("ops_error_logs");
}
Ok(())
}
async fn import_request_time_buckets(
source: &SqlitePool,
target: &mut Transaction<'_, Postgres>,
report: &mut ImportSqliteReport,
) -> Result<(), ImportSqliteError> {
let mut rows =
sqlx::query("select * from usage_time_buckets order by bucket_start").fetch(source);
while let Some(row) = rows.try_next().await? {
let request_count: i64 = row.try_get("request_count")?;
let error_count: i64 = row.try_get("error_count")?;
let min_latency_ms: i64 = row.try_get("min_latency_ms")?;
sqlx::query(
"insert into request_time_buckets
(bucket_start, provider, account_id, model, service_tier, success_count,
error_count, input_tokens, output_tokens, cached_tokens,
first_token_latency_sum, first_token_latency_count, latency_sum,
latency_count, max_latency_ms, min_latency_ms, updated_at)
values ($1, 'openai', $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
$12, $13, $14, $15, $16)",
)
.bind(parse_timestamp(
"usage_time_buckets",
"bucket_start",
row.try_get("bucket_start")?,
)?)
.bind(normalized_dimension(row.try_get("account_id")?))
.bind(normalized_dimension(row.try_get("model")?))
.bind(normalized_dimension(row.try_get("service_tier")?))
.bind(request_count.saturating_sub(error_count).max(0))
.bind(error_count)
.bind(row.try_get::<i64, _>("input_tokens")?)
.bind(row.try_get::<i64, _>("output_tokens")?)
.bind(row.try_get::<i64, _>("cached_tokens")?)
.bind(row.try_get::<i64, _>("first_token_latency_sum")?)
.bind(row.try_get::<i64, _>("first_token_latency_count")?)
.bind(row.try_get::<i64, _>("latency_sum")?)
.bind(row.try_get::<i64, _>("latency_count")?)
.bind(row.try_get::<i64, _>("max_latency_ms")?)
.bind((min_latency_ms != 0).then_some(min_latency_ms))
.bind(parse_timestamp(
"usage_time_buckets",
"updated_at",
row.try_get("updated_at")?,
)?)
.execute(&mut **target)
.await?;
report.add_imported("request_time_buckets");
}
Ok(())
}
fn optional_status(status: Option<i64>) -> Option<i32> {
status.and_then(|status| i32::try_from(status).ok())
}
fn metadata_string(metadata: &Value, keys: &[&str]) -> Option<String> {
keys.iter().find_map(|key| {
metadata
.get(*key)
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
})
}
fn metadata_nonnegative(metadata: &Value, keys: &[&str]) -> Option<i64> {
keys.iter()
.find_map(|key| metadata.get(*key).and_then(Value::as_i64))
.filter(|value| *value >= 0)
}
fn usage_number(metadata: &Value, keys: &[&str]) -> Option<i64> {
metadata
.get("usage")
.and_then(|usage| metadata_nonnegative(usage, keys))
.or_else(|| metadata_nonnegative(metadata, keys))
}
fn strip_promoted_success_fields(metadata: &mut Value) {
let Some(object) = metadata.as_object_mut() else {
return;
};
for key in [
"usage",
"route",
"apiKind",
"requestedModel",
"requested_model",
"upstreamModel",
"upstream_model",
"serviceTier",
"service_tier",
"statusCode",
"transport",
"attemptIndex",
"responseId",
"upstreamRequestId",
"openaiRequestId",
"firstTokenMs",
"first_token_ms",
] {
object.remove(key);
}
}
+8
View File
@@ -0,0 +1,8 @@
//! 进程装配、共享状态与后台任务。
pub mod config;
pub mod import_sqlite;
pub mod services;
pub mod shutdown;
pub mod state;
pub mod tasks;
+637
View File
@@ -0,0 +1,637 @@
//! 运行时服务集合。
use std::{env, error::Error, future::IntoFuture, sync::Arc, time::Duration};
use axum::Router;
use crate::{
auth::{
service::SessionService,
store::{PgAdminUserStore, RedisAdminSessionStore},
},
bootstrap::config::AppConfig,
dispatch::{
affinity::{RedisSessionAffinityStore, SessionAffinityService},
service::{ResponseDispatchService, ResponseDispatchServiceParts},
},
fleet::{
cookies::PgCookieStore,
manage::{AccountManageService, AccountManageServiceParts},
pool::{AccountPoolOptions, AccountPoolService, AccountPoolStaticSettings},
refresh::{
RedisRefreshLeaseStore, RefreshPolicy, RuntimeRefreshPolicy, TokenRefreshService,
},
store::{AccountStore, PgAccountStore},
},
infra::{
database::connect,
identity::AccountPseudonymizer,
logging::{init_tracing, LogError, LogGuard, RotationConfig, TracingConfig},
paths::{ensure_data_dir, load_or_create_identity_secret},
redis::RedisConnection,
},
keys::{manage::KeyManageService, service::KeyVerifier, store::PgClientKeyStore},
models::{
service::{ModelService, ModelServiceError},
store::{ModelSnapshotStore, RedisModelSnapshotStore},
types::ModelConfig,
},
settings::{service::SettingsService, SettingsSnapshot},
telemetry::{
account_usage::query::AccountUsageQueryService,
account_usage::store::{AccountUsageStore, PgAccountUsageStore},
buckets::store::PgRequestBucketStore,
ops::query::OpsQueryService,
ops::store::PgOpsErrorLogStore,
recorder::Recorder,
usage::query::UsageQueryService,
usage::store::{PgUsageRecordStore, DEFAULT_USAGE_RECORD_CAPTURE_BODY},
},
upstream::openai::{
fingerprint::{Fingerprint, PgFingerprintStore, RuntimeFingerprint},
token_client::{default_openai_token_client, OpenAiTokenClient, TokenClientConfig},
transport::{
build_reqwest_client, tls::CustomCaError, CodexBackendClient, CodexModelCatalogClient,
CodexWebSocketPool, CodexWebSocketPoolConfig,
},
},
};
const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS";
/// 运行时热路径内存状态初始化错误。
#[derive(Debug, thiserror::Error)]
pub enum RuntimeServiceInitializationError {
#[error("failed to initialize model catalog runtime cache: {0}")]
Models(#[from] ModelServiceError),
}
// ============================================================================
// BackgroundTaskStores
// ============================================================================
/// 后台任务需要的具体存储适配器集合。
#[derive(Clone)]
pub struct BackgroundTaskStores {
pub redis: RedisConnection,
pub accounts: PgAccountStore,
pub admin_users: PgAdminUserStore,
pub admin_sessions: RedisAdminSessionStore,
pub cookies: PgCookieStore,
pub fingerprints: PgFingerprintStore,
pub session_affinity: RedisSessionAffinityStore,
pub refresh_leases: RedisRefreshLeaseStore,
pub model_snapshots: RedisModelSnapshotStore,
pub client_keys: PgClientKeyStore,
pub usage_records: PgUsageRecordStore,
pub ops_errors: PgOpsErrorLogStore,
pub account_usage: PgAccountUsageStore,
pub request_buckets: PgRequestBucketStore,
}
// ============================================================================
// Services struct
// ============================================================================
/// 运行时服务集合。
#[derive(Clone)]
pub struct Services {
pub database: sqlx::PgPool,
pub redis: RedisConnection,
pub models: Arc<ModelService>,
pub accounts: Arc<dyn AccountStore>,
pub client_keys: Arc<KeyVerifier>,
pub admin_client_keys: Arc<KeyManageService>,
pub admin_sessions: Arc<SessionService>,
pub settings: Arc<SettingsService>,
pub refresh_policy: RuntimeRefreshPolicy,
pub admin_accounts: Arc<AccountManageService>,
pub usage_records: Arc<UsageQueryService>,
pub ops_errors: Arc<OpsQueryService>,
pub usage: Arc<AccountUsageQueryService>,
pub account_pool: Arc<AccountPoolService>,
pub token_refresh: Arc<TokenRefreshService<OpenAiTokenClient>>,
pub responses: Arc<ResponseDispatchService>,
pub session_affinity: Arc<SessionAffinityService>,
pub codex: Arc<CodexBackendClient>,
pub websocket_pool: Option<Arc<CodexWebSocketPool>>,
pub fingerprint: RuntimeFingerprint,
pub account_pseudonymizer: Arc<AccountPseudonymizer>,
pub background_tasks: BackgroundTaskStores,
pub(crate) account_pool_static: AccountPoolStaticSettings,
}
/// 使用记录运行选项。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UsageRecordOptions {
pub enabled: bool,
pub capture_body: bool,
}
impl UsageRecordOptions {
pub fn from_config(config: &AppConfig) -> Self {
Self {
enabled: config.telemetry.enabled,
capture_body: DEFAULT_USAGE_RECORD_CAPTURE_BODY,
}
}
}
impl Services {
pub fn new(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
) -> Self {
Self::try_new(config, stores, fingerprint)
.expect("failed to build runtime services with configured TLS transport")
}
pub fn try_new(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
) -> Result<Self, CustomCaError> {
Self::try_with_usage_record_options(
config,
stores,
fingerprint,
UsageRecordOptions::from_config(config),
)
}
pub fn try_with_usage_record_options(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
usage_record_options: UsageRecordOptions,
) -> Result<Self, CustomCaError> {
let mut identity_secret = [0u8; 32];
rand::Rng::fill_bytes(&mut rand::rng(), &mut identity_secret);
Self::try_with_identity_secret_and_usage_record_options(
config,
stores,
fingerprint,
identity_secret,
usage_record_options,
)
}
fn try_with_identity_secret_and_usage_record_options(
config: &AppConfig,
stores: BackgroundTaskStores,
fingerprint: RuntimeFingerprint,
identity_secret: [u8; 32],
usage_record_options: UsageRecordOptions,
) -> Result<Self, CustomCaError> {
let account_pseudonymizer = Arc::new(AccountPseudonymizer::new(identity_secret));
let account_identity = Arc::new(crate::dispatch::affinity::AccountIdentityService::new(
(*account_pseudonymizer).clone(),
));
let database = stores.accounts.pool().clone();
let redis = stores.redis.clone();
let account_store = Arc::new(stores.accounts.clone()) as Arc<dyn AccountStore>;
let account_usage_store_trait =
Arc::new(stores.account_usage.clone()) as Arc<dyn AccountUsageStore>;
let websocket_pool = config.ws_pool.enabled.then(|| {
Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig {
enabled: true,
max_age: std::time::Duration::from_millis(config.ws_pool.max_age_ms),
max_per_account: config.ws_pool.max_per_account,
initial_event_timeout: (config.ws_pool.initial_event_timeout_ms > 0).then(|| {
std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms)
}),
..CodexWebSocketPoolConfig::default()
}))
});
let codex = {
let websocket_initial_event_timeout = (config.ws_pool.initial_event_timeout_ms > 0)
.then(|| std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms));
let client = CodexBackendClient::new(
build_reqwest_client(config.tls.force_http11)?,
config.api.base_url.clone(),
fingerprint.clone(),
)
.with_websocket_initial_event_timeout(websocket_initial_event_timeout);
if let Some(pool) = &websocket_pool {
Arc::new(client.with_websocket_pool(pool.clone()))
} else {
Arc::new(client)
}
};
let usage_records = Arc::new(UsageQueryService::new(stores.usage_records.clone()));
let ops_errors = Arc::new(OpsQueryService::new(stores.ops_errors.clone()));
let recorder = Arc::new(Recorder::new(
stores.usage_records.clone(),
stores.ops_errors.clone(),
usage_record_options.enabled,
usage_record_options.capture_body,
));
let settings_snapshot = settings_snapshot_from_config(config);
let account_pool_options = account_pool_options_from_config(config);
let account_pool_static = AccountPoolStaticSettings {
skip_quota_limited: account_pool_options.skip_quota_limited,
tier_priority: account_pool_options.tier_priority.clone(),
};
let account_pool = Arc::new(AccountPoolService::new(
account_store.clone(),
account_usage_store_trait,
account_pool_options,
settings_snapshot.request_interval_ms,
));
let refresh_policy = RuntimeRefreshPolicy::new(RefreshPolicy {
refresh_margin_seconds: config.auth.refresh_margin_seconds,
refresh_concurrency: config.auth.refresh_concurrency,
});
let settings = Arc::new(SettingsService::new(
settings_snapshot,
stores.accounts.pool().clone(),
));
let admin_sessions = Arc::new(SessionService::new(
stores.admin_users.clone(),
stores.admin_sessions.clone(),
config.admin.default_username.clone(),
config.admin.session_ttl_minutes,
));
let admin_client_keys = Arc::new(KeyManageService::new(stores.client_keys.clone()));
let client_keys = Arc::new(KeyVerifier::new(stores.client_keys.clone()));
let token_client = default_openai_token_client(token_client_config(config));
let token_refresh = Arc::new(
TokenRefreshService::new(
stores.accounts.clone(),
refresh_policy.clone(),
token_client.clone(),
)
.with_refresh_lease_store(stores.refresh_leases.clone()),
);
let upstream_client: Arc<dyn CodexModelCatalogClient> = codex.clone();
let store: Arc<dyn ModelSnapshotStore> = Arc::new(stores.model_snapshots.clone());
let models = Arc::new(ModelService::new(
ModelConfig {
model_aliases: config.model_aliases.clone(),
},
Some(store),
Some(upstream_client),
));
let admin_accounts = Arc::new(AccountManageService::new(AccountManageServiceParts {
store: stores.accounts.clone(),
cookies: stores.cookies.clone(),
codex: codex.clone(),
models: models.clone(),
account_pool: account_pool.clone(),
token_refresher: Arc::new(token_client),
refresh_leases: stores.refresh_leases.clone(),
oauth: crate::fleet::manage::oauth::AccountOAuthService::new(
reqwest::Client::new(),
config.auth.oauth_client_id.clone(),
config.auth.oauth_token_endpoint.clone(),
),
refresh_policy: refresh_policy.clone(),
account_pseudonymizer: account_pseudonymizer.clone(),
}));
let usage = Arc::new(AccountUsageQueryService::new(stores.account_usage.clone()));
let session_affinity =
Arc::new(SessionAffinityService::new(stores.session_affinity.clone()));
let cloudflare_recovery =
crate::dispatch::recovery::cloudflare::CloudflareRecovery::new(stores.cookies.clone());
let responses = Arc::new(ResponseDispatchService::new(ResponseDispatchServiceParts {
account_pool: account_pool.clone(),
models: models.clone(),
codex: codex.clone(),
session_affinity: session_affinity.clone(),
recorder,
account_identity,
cloudflare: cloudflare_recovery,
}));
Ok(Self {
database,
redis,
models,
accounts: account_store,
client_keys,
admin_client_keys,
admin_sessions,
settings,
refresh_policy,
admin_accounts,
usage_records,
ops_errors,
usage,
account_pool,
token_refresh,
responses,
session_affinity,
codex,
websocket_pool,
fingerprint,
account_pseudonymizer,
background_tasks: stores,
account_pool_static,
})
}
/// 初始化请求热路径依赖的内存状态。
pub async fn initialize_hot_path_state(&self) -> Result<(), RuntimeServiceInitializationError> {
self.models.reload_from_store().await?;
Ok(())
}
}
impl From<&Services> for crate::api::router::ApiServices {
fn from(services: &Services) -> Self {
Self {
health_probe: Arc::new(crate::bootstrap::state::RuntimeHealthProbe::new(
services.database.clone(),
services.redis.clone(),
)),
models: services.models.clone(),
accounts: services.accounts.clone(),
client_keys: services.client_keys.clone(),
admin_client_keys: services.admin_client_keys.clone(),
admin_sessions: services.admin_sessions.clone(),
settings: services.settings.clone(),
admin_accounts: services.admin_accounts.clone(),
usage_records: services.usage_records.clone(),
ops_errors: services.ops_errors.clone(),
usage: services.usage.clone(),
account_pool: services.account_pool.clone(),
token_refresh: services.token_refresh.clone(),
responses: services.responses.clone(),
session_affinity: services.session_affinity.clone(),
fingerprint: services.fingerprint.clone(),
process_control: Arc::new(crate::bootstrap::shutdown::RuntimeProcessControl),
}
}
}
impl From<&Services> for crate::api::router::AppState {
fn from(services: &Services) -> Self {
Self {
services: crate::api::router::ApiServices::from(services),
}
}
}
impl From<Services> for crate::api::router::AppState {
fn from(services: Services) -> Self {
Self::from(&services)
}
}
fn token_client_config(config: &AppConfig) -> TokenClientConfig {
TokenClientConfig {
client_id: config.auth.oauth_client_id.clone(),
token_endpoint: config.auth.oauth_token_endpoint.clone(),
}
}
pub fn settings_snapshot_from_config(config: &AppConfig) -> SettingsSnapshot {
SettingsSnapshot {
model_aliases: config.model_aliases.clone(),
refresh_margin_seconds: config.auth.refresh_margin_seconds,
refresh_concurrency: config.auth.refresh_concurrency,
max_concurrent_per_account: config.auth.max_concurrent_per_account,
request_interval_ms: config.auth.request_interval_ms,
rotation_strategy: config.auth.rotation_strategy.clone(),
}
}
pub fn apply_settings_to_config(config: &mut AppConfig, settings: &SettingsSnapshot) {
config.model_aliases = settings.model_aliases.clone();
config.auth.refresh_margin_seconds = settings.refresh_margin_seconds;
config.auth.refresh_concurrency = settings.refresh_concurrency;
config.auth.max_concurrent_per_account = settings.max_concurrent_per_account;
config.auth.request_interval_ms = settings.request_interval_ms;
config.auth.rotation_strategy = settings.rotation_strategy.clone();
}
pub fn fingerprint_from_config(
config: &crate::bootstrap::config::FingerprintConfig,
) -> Fingerprint {
Fingerprint {
originator: config.originator.clone(),
app_version: config.app_version.clone(),
build_number: config.build_number.clone(),
platform: config.platform.clone(),
arch: config.arch.clone(),
chromium_version: config.chromium_version.clone(),
user_agent_template: config.user_agent_template.clone(),
default_headers: config
.default_headers
.iter()
.map(|header| (header.name.clone(), header.value.clone()))
.collect(),
header_order: config.header_order.clone(),
updated_at: None,
}
}
pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions {
AccountPoolStaticSettings {
skip_quota_limited: config.quota.skip_exhausted,
tier_priority: config.auth.tier_priority.clone(),
}
.pool_options(&settings_snapshot_from_config(config))
}
pub async fn run() -> Result<(), Box<dyn Error + Send + Sync>> {
wait_for_scheduled_restart().await;
let config = AppConfig::load()?;
let host = config.server.host.clone();
let port = config.server.port;
let log_guard = init_logging(&config)?;
let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?;
let (router, task_coordinator) = build_router(config).await?;
let serve_result = {
let (graceful_shutdown_tx, graceful_shutdown_rx) = tokio::sync::oneshot::channel();
let serve = axum::serve(
listener,
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(async move {
let _ = graceful_shutdown_rx.await;
})
.into_future();
tokio::pin!(serve);
tokio::select! {
result = &mut serve => result,
() = crate::bootstrap::shutdown::shutdown_signal() => {
let _ = graceful_shutdown_tx.send(());
tokio::select! {
result = &mut serve => result,
() = tokio::time::sleep(HTTP_DRAIN_TIMEOUT) => {
tracing::warn!(
timeout_secs = HTTP_DRAIN_TIMEOUT.as_secs(),
"HTTP graceful shutdown timed out; cancelling remaining connections"
);
Ok(())
}
() = crate::bootstrap::shutdown::shutdown_signal() => {
tracing::warn!("received a second shutdown signal; cancelling remaining connections");
Ok(())
}
}
}
}
};
task_coordinator.shutdown().await;
serve_result?;
if let Some(executable_path) = crate::bootstrap::shutdown::restart_executable_path() {
tracing::info!(
executable = %executable_path.display(),
"正在以更新后的二进制替换当前进程"
);
drop(log_guard);
exec_replacement_process(executable_path)?;
}
Ok(())
}
const HTTP_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
async fn build_router(
mut config: AppConfig,
) -> Result<
(
Router,
crate::bootstrap::tasks::coordinator::TaskCoordinator,
),
Box<dyn Error + Send + Sync>,
> {
let pool = connect(&config.database.url).await?;
let redis = RedisConnection::connect(&config.redis.url, "cpr").await?;
let settings =
SettingsService::load_or_initialize(settings_snapshot_from_config(&config), &pool).await?;
apply_settings_to_config(&mut config, &settings);
let fingerprint_store = PgFingerprintStore::new(pool.clone());
let stores = BackgroundTaskStores {
redis: redis.clone(),
accounts: PgAccountStore::new(pool.clone()),
admin_users: PgAdminUserStore::new(pool.clone()),
admin_sessions: RedisAdminSessionStore::new(redis.clone()),
cookies: PgCookieStore::new(pool.clone()),
fingerprints: fingerprint_store.clone(),
session_affinity: RedisSessionAffinityStore::new(redis.clone()),
refresh_leases: RedisRefreshLeaseStore::new(redis.clone()),
model_snapshots: RedisModelSnapshotStore::new(redis.clone()),
client_keys: PgClientKeyStore::new(pool.clone()),
usage_records: PgUsageRecordStore::new(pool.clone()),
ops_errors: PgOpsErrorLogStore::new(pool.clone()),
account_usage: PgAccountUsageStore::new(pool.clone()),
request_buckets: PgRequestBucketStore::new(pool),
};
config.admin.validate_default_password()?;
let default_admin_password = config.admin.default_password.clone();
let data_dir = ensure_data_dir()?;
let identity_secret = load_or_create_identity_secret(&data_dir)?;
let default_fingerprint = fingerprint_from_config(&config.fingerprint);
let runtime_fingerprint = fingerprint_store
.ensure_current_seed(&default_fingerprint)
.await?;
let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint);
let runtime_config = crate::bootstrap::state::RuntimeConfig::from(config.clone());
let services = Services::try_with_identity_secret_and_usage_record_options(
&config,
stores,
runtime_fingerprint,
identity_secret,
UsageRecordOptions::from_config(&config),
)?;
services.initialize_hot_path_state().await?;
let created_default_admin = services
.admin_sessions
.ensure_default_admin(&default_admin_password)
.await?;
tracing::info!(
created = created_default_admin,
"默认管理员账号已完成初始化检查"
);
let restored_accounts = services.account_pool.restore_from_store().await?;
tracing::info!(
count = restored_accounts,
"运行时账号池已从 PostgreSQL 恢复"
);
let task_coordinator =
crate::bootstrap::tasks::coordinator::TaskCoordinator::start(&runtime_config, &services);
let app_state = crate::api::AppState::from(&services);
Ok((
crate::api::router::router().with_state(app_state),
task_coordinator,
))
}
fn init_logging(config: &AppConfig) -> Result<LogGuard, LogError> {
const MEBIBYTE: u64 = 1024 * 1024;
let file = if config.logging.file.enabled {
let max_file_size_bytes = config
.logging
.file
.max_file_size_mb
.checked_mul(MEBIBYTE)
.ok_or(LogError::InvalidConfiguration(
"logging.file.max_file_size_mb is too large",
))?;
Some(RotationConfig::new(
&config.logging.file.directory,
config.logging.file.retention_days,
max_file_size_bytes,
config.logging.file.max_files,
))
} else {
None
};
init_tracing(&TracingConfig::new(
&config.logging.level,
config.logging.stdout,
file,
))
}
async fn wait_for_scheduled_restart() {
let Some(delay_ms) = env::var(RESTART_DELAY_ENV)
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| *value > 0)
else {
return;
};
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
}
#[cfg(unix)]
fn exec_replacement_process(
executable_path: std::path::PathBuf,
) -> Result<(), Box<dyn Error + Send + Sync>> {
use std::os::unix::process::CommandExt;
use std::process::Command;
let error = Command::new(executable_path)
.args(env::args_os().skip(1))
.exec();
Err(Box::new(error))
}
#[cfg(not(unix))]
fn exec_replacement_process(
_executable_path: std::path::PathBuf,
) -> Result<(), Box<dyn Error + Send + Sync>> {
Err(Box::new(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"process exec restart is only supported on Unix",
)))
}
@@ -15,6 +15,23 @@ static SHUTDOWN_REQUESTS: OnceLock<broadcast::Sender<()>> = OnceLock::new();
static RESTART_REQUESTED: AtomicBool = AtomicBool::new(false);
static RESTART_EXECUTABLE_PATH: OnceLock<PathBuf> = OnceLock::new();
#[derive(Debug, Clone, Copy, Default)]
pub struct RuntimeProcessControl;
impl crate::api::router::ProcessControl for RuntimeProcessControl {
fn request_shutdown(&self) {
request_shutdown();
}
fn request_restart(&self, executable_path: PathBuf) {
request_process_restart(executable_path);
}
fn subscribe_shutdown(&self) -> Receiver<()> {
shutdown_subscription()
}
}
/// 等待进程关闭信号(Ctrl+C 或 SIGTERM)。
pub async fn shutdown_signal() {
let mut requested = shutdown_sender().subscribe();
+51
View File
@@ -0,0 +1,51 @@
//! 运行时共享状态。
use async_trait::async_trait;
use crate::{
api::router::HealthProbe, bootstrap::config::AppConfig, infra::redis::RedisConnection,
};
/// 运行时配置镜像(从 AppConfig 衍生)。
#[derive(Debug, Clone)]
pub struct RuntimeConfig {
pub admin: crate::bootstrap::config::AdminConfig,
pub auth: crate::bootstrap::config::AuthConfig,
pub quota: crate::bootstrap::config::QuotaConfig,
}
/// PostgreSQL 与 Redis 的进程健康检查实现。
#[derive(Clone)]
pub struct RuntimeHealthProbe {
database: sqlx::PgPool,
redis: RedisConnection,
}
impl RuntimeHealthProbe {
pub fn new(database: sqlx::PgPool, redis: RedisConnection) -> Self {
Self { database, redis }
}
}
#[async_trait]
impl HealthProbe for RuntimeHealthProbe {
async fn check(&self) -> Result<(), String> {
crate::infra::database::ping(&self.database)
.await
.map_err(|error| format!("PostgreSQL: {error}"))?;
self.redis
.ping()
.await
.map_err(|error| format!("Redis: {error}"))
}
}
impl From<AppConfig> for RuntimeConfig {
fn from(config: AppConfig) -> Self {
Self {
admin: config.admin.clone(),
auth: config.auth.clone(),
quota: config.quota,
}
}
}
@@ -4,7 +4,7 @@ use std::future::Future;
use chrono::{DateTime, Utc};
use crate::upstream::accounts::cookies::{SqliteCookieStore, SqliteCookieStoreError};
use crate::fleet::cookies::{PgCookieStore, PgCookieStoreError};
use super::{
cleanup::{CleanupTask, CleanupTaskMessages, ExpiredCleanupStore},
@@ -22,12 +22,12 @@ const MESSAGES: CleanupTaskMessages = CleanupTaskMessages {
/// Cookie 清理任务。
pub struct CookieCleanupTask {
task: CleanupTask<SqliteCookieStore>,
task: CleanupTask<PgCookieStore>,
}
impl CookieCleanupTask {
/// 构造默认 Cookie 清理任务。
pub fn new(store: SqliteCookieStore) -> Self {
pub fn new(store: PgCookieStore) -> Self {
Self {
task: CleanupTask::new(store, DEFAULT_INTERVAL_SECS, MESSAGES),
}
@@ -39,18 +39,18 @@ impl CookieCleanupTask {
}
/// 执行一次清理。
pub async fn cleanup_once(&self) -> Result<u64, SqliteCookieStoreError> {
pub async fn cleanup_once(&self) -> Result<u64, PgCookieStoreError> {
self.task.cleanup_once().await
}
/// 在指定时间点执行一次清理。
pub async fn cleanup_once_at(&self, now: DateTime<Utc>) -> Result<u64, SqliteCookieStoreError> {
pub async fn cleanup_once_at(&self, now: DateTime<Utc>) -> Result<u64, PgCookieStoreError> {
self.task.cleanup_once_at(now).await
}
}
impl ExpiredCleanupStore for SqliteCookieStore {
type Error = SqliteCookieStoreError;
impl ExpiredCleanupStore for PgCookieStore {
type Error = PgCookieStoreError;
fn delete_expired_at(
&self,
@@ -1,6 +1,6 @@
//! 后台任务协调器。
use std::{path::PathBuf, sync::Arc, time::Duration};
use std::{future::Future, path::PathBuf, sync::Arc, time::Duration};
use tokio::task::JoinHandle;
@@ -12,8 +12,7 @@ use super::{
},
model_refresh::ModelRefreshTask,
quota_refresh::QuotaRefreshTask,
session_affinity_cleanup::SessionAffinityCleanupTask,
session_cleanup::SessionCleanupTask,
retention_trim::RetentionTrimTask,
token_refresh::TokenRefreshTask,
};
@@ -31,10 +30,10 @@ impl TaskCoordinator {
/// 启动所有后台任务。
pub(crate) fn start(
config: &crate::runtime::state::RuntimeConfig,
services: &crate::runtime::services::Services,
config: &crate::bootstrap::state::RuntimeConfig,
services: &crate::bootstrap::services::Services,
) -> Self {
let mut coordinator = TaskCoordinator::default();
let mut coordinator = Self::start_settings_subscriptions(services);
let stores = &services.background_tasks;
coordinator.push(
@@ -42,26 +41,34 @@ impl TaskCoordinator {
CookieCleanupTask::new(stores.cookies.clone()).start(),
);
coordinator.push(
"session_cleanup",
SessionCleanupTask::new(
stores.admin_sessions.clone(),
config.admin.session_cleanup_interval_secs,
)
.start(),
);
coordinator.push(
"session_affinity_cleanup",
SessionAffinityCleanupTask::new(
stores.session_affinity.clone(),
config.admin.session_cleanup_interval_secs,
"retention_trim",
RetentionTrimTask::new(
stores.usage_records.clone(),
stores.ops_errors.clone(),
stores.request_buckets.clone(),
stores.fingerprints.clone(),
)
.start(),
);
coordinator.push(
"model_refresh",
ModelRefreshTask::new(services.models.clone(), services.account_pool.clone())
.with_installation_id(services.installation_id.clone())
.start(),
ModelRefreshTask::new(
services.models.clone(),
services.account_pool.clone(),
services.account_pseudonymizer.clone(),
)
.start(),
);
coordinator.push(
"model_etag_refresh",
SchedulerHandle::spawn(
ModelRefreshTask::new(
services.models.clone(),
services.account_pool.clone(),
services.account_pseudonymizer.clone(),
)
.run_etag_refreshes(),
),
);
if config.auth.refresh_enabled {
coordinator.push(
@@ -73,7 +80,9 @@ impl TaskCoordinator {
"quota_refresh",
QuotaRefreshTask::with_intervals(
stores.accounts.clone(),
stores.account_usage.clone(),
services.codex.clone(),
services.account_pseudonymizer.clone(),
config
.quota
.refresh_interval_minutes
@@ -81,21 +90,21 @@ impl TaskCoordinator {
.max(1),
30 * 60,
)
.with_installation_id(services.installation_id.clone())
.with_cookie_store(stores.cookies.clone())
.with_account_pool(services.account_pool.clone())
.start(),
);
coordinator.push(
"fingerprint_update",
coordinator.push("fingerprint_update", {
let fingerprint = services.fingerprint.snapshot();
start_fingerprint_update_task(
Some(stores.fingerprints.clone()),
stores.fingerprints.clone(),
services.fingerprint.clone(),
CODEX_DESKTOP_APPCAST_URL.to_string(),
PathBuf::from(DEFAULT_EXTRACTED_FINGERPRINT_PATH),
services.fingerprint.app_version.clone(),
services.fingerprint.build_number.clone(),
),
);
fingerprint.app_version,
fingerprint.build_number,
)
});
if let Some(pool) = &services.websocket_pool {
coordinator.push(
"websocket_pool",
@@ -106,12 +115,55 @@ impl TaskCoordinator {
coordinator
}
/// 启动各领域的设置订阅任务。
pub fn start_settings_subscriptions(services: &crate::bootstrap::services::Services) -> Self {
let mut coordinator = TaskCoordinator::default();
coordinator.push(
"model_settings",
SchedulerHandle::spawn(
services
.models
.clone()
.subscribe_settings(services.settings.subscribe()),
),
);
coordinator.push(
"account_pool_settings",
SchedulerHandle::spawn(services.account_pool.clone().subscribe_settings(
services.settings.subscribe(),
services.account_pool_static.clone(),
)),
);
coordinator.push(
"refresh_policy_settings",
SchedulerHandle::spawn(
services
.refresh_policy
.clone()
.subscribe_settings(services.settings.subscribe()),
),
);
coordinator
}
/// 关闭所有后台任务。
pub async fn shutdown(self) {
tracing::info!("正在关闭后台任务");
for (name, handle) in self.handles {
let shutdown_tasks = self.handles.into_iter().map(|(name, handle)| async move {
handle.shutdown().await;
tracing::info!(task = name, "后台任务已停止");
});
if tokio::time::timeout(
COORDINATOR_SHUTDOWN_TIMEOUT,
futures::future::join_all(shutdown_tasks),
)
.await
.is_err()
{
tracing::warn!(
timeout_secs = COORDINATOR_SHUTDOWN_TIMEOUT.as_secs(),
"后台任务并行关闭超时"
);
}
tracing::info!("所有后台任务已停止");
}
@@ -125,10 +177,21 @@ pub enum SchedulerHandle {
handle: JoinHandle<()>,
},
/// 关闭上游 WebSocket 连接池。
WebSocketPool(Arc<crate::upstream::transport::CodexWebSocketPool>),
WebSocketPool(Arc<crate::upstream::openai::transport::CodexWebSocketPool>),
}
impl SchedulerHandle {
fn spawn(future: impl Future<Output = ()> + Send + 'static) -> Self {
let (shutdown_tx, mut shutdown_rx) = tokio::sync::mpsc::channel(1);
let handle = tokio::spawn(async move {
tokio::select! {
() = future => {}
_ = shutdown_rx.recv() => {}
}
});
Self::new(shutdown_tx, handle)
}
/// 使用关闭 channel 构造句柄。
pub(crate) fn new(shutdown_tx: tokio::sync::mpsc::Sender<()>, handle: JoinHandle<()>) -> Self {
Self::Channel {
@@ -138,7 +201,9 @@ impl SchedulerHandle {
}
/// 使用 WebSocket 连接池构造句柄。
pub fn from_websocket_pool(pool: Arc<crate::upstream::transport::CodexWebSocketPool>) -> Self {
pub fn from_websocket_pool(
pool: Arc<crate::upstream::openai::transport::CodexWebSocketPool>,
) -> Self {
Self::WebSocketPool(pool)
}
@@ -160,6 +225,7 @@ impl SchedulerHandle {
}
const TASK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
const COORDINATOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(6);
async fn wait_for_task_shutdown(handle: JoinHandle<()>) {
let mut handle = handle;
@@ -2,9 +2,11 @@
use std::path::PathBuf;
use tracing::warn;
use tracing::{info, warn};
use crate::upstream::fingerprint::{FingerprintRepository, UpdateChecker, APPCAST_POLL_INTERVAL};
use crate::upstream::openai::fingerprint::{
PgFingerprintStore, RuntimeFingerprint, UpdateChecker, APPCAST_POLL_INTERVAL,
};
use super::{
coordinator::SchedulerHandle,
@@ -19,14 +21,15 @@ pub const DEFAULT_EXTRACTED_FINGERPRINT_PATH: &str = "data/extracted-fingerprint
/// 启动指纹自动更新后台任务。
pub fn start_fingerprint_update_task(
repository: Option<FingerprintRepository>,
store: PgFingerprintStore,
runtime_fingerprint: RuntimeFingerprint,
appcast_url: String,
extracted_fingerprint_path: PathBuf,
current_version: String,
current_build: String,
) -> SchedulerHandle {
let update_checker = UpdateChecker::with_client(
repository,
store,
reqwest::Client::new(),
appcast_url,
extracted_fingerprint_path,
@@ -36,6 +39,7 @@ pub fn start_fingerprint_update_task(
spawn_periodic_task(
FingerprintUpdateTask {
checker: update_checker,
runtime_fingerprint,
first_tick: true,
},
PeriodicTaskConfig::new(
@@ -48,6 +52,7 @@ pub fn start_fingerprint_update_task(
struct FingerprintUpdateTask {
checker: UpdateChecker,
runtime_fingerprint: RuntimeFingerprint,
first_tick: bool,
}
@@ -55,11 +60,24 @@ impl PeriodicTaskRunner for FingerprintUpdateTask {
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
Box::pin(async move {
let first_tick = std::mem::take(&mut self.first_tick);
if let Err(error) = self.checker.check_and_apply_update().await {
if first_tick {
warn!(error = %error, "fingerprint 首次版本检查失败");
} else {
warn!(error = %error, "fingerprint 定期版本检查失败");
match self.checker.check_and_apply_update().await {
Ok(Some(fingerprint)) => {
let app_version = fingerprint.app_version.clone();
let build_number = fingerprint.build_number.clone();
self.runtime_fingerprint.replace(fingerprint);
info!(
app_version = %app_version,
build_number = %build_number,
"fingerprint 运行时版本已更新"
);
}
Ok(None) => {}
Err(error) => {
if first_tick {
warn!(error = %error, "fingerprint 首次版本检查失败");
} else {
warn!(error = %error, "fingerprint 定期版本检查失败");
}
}
}
})
@@ -7,6 +7,5 @@ pub mod fingerprint_update;
pub mod model_refresh;
mod periodic;
pub mod quota_refresh;
pub mod session_affinity_cleanup;
pub mod session_cleanup;
pub mod retention_trim;
pub mod token_refresh;
@@ -7,8 +7,9 @@ use tokio::sync::mpsc;
use tokio::time::sleep;
use tracing::{info, warn};
use crate::upstream::accounts::pool::RuntimeAccountPoolService;
use crate::upstream::models::service::{
use crate::fleet::pool::AccountPoolService;
use crate::infra::identity::AccountPseudonymizer;
use crate::models::service::{
ModelRefreshPlanAccount, ModelRefreshResult, ModelService, ModelServiceError,
};
@@ -20,12 +21,12 @@ use super::{
/// 模型刷新任务接线器。
pub struct ModelRefreshTask {
model_service: Arc<ModelService>,
account_pool: Arc<RuntimeAccountPoolService>,
account_pool: Arc<AccountPoolService>,
refresh_interval_secs: u64,
initial_delay_ms: u64,
retry_delay_ms: u64,
max_retries: u32,
installation_id: Option<String>,
account_pseudonymizer: Arc<AccountPseudonymizer>,
}
const DEFAULT_REFRESH_INTERVAL_SECS: u64 = 3600;
@@ -37,7 +38,8 @@ impl ModelRefreshTask {
/// 构造默认任务。
pub fn new(
model_service: Arc<ModelService>,
account_pool: Arc<RuntimeAccountPoolService>,
account_pool: Arc<AccountPoolService>,
account_pseudonymizer: Arc<AccountPseudonymizer>,
) -> Self {
Self {
model_service,
@@ -46,16 +48,10 @@ impl ModelRefreshTask {
initial_delay_ms: INITIAL_DELAY_MS,
retry_delay_ms: RETRY_DELAY_MS,
max_retries: MAX_RETRIES,
installation_id: None,
account_pseudonymizer,
}
}
/// 设置 Codex installation id。
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
self.installation_id = installation_id.filter(|id| !id.trim().is_empty());
self
}
/// 启动后台刷新任务。
pub fn start(self) -> SchedulerHandle {
let config = PeriodicTaskConfig::new(
@@ -67,6 +63,27 @@ impl ModelRefreshTask {
spawn_periodic_task(self, config)
}
/// 在 Responses 响应声明模型目录 ETag 变化时异步刷新。
pub async fn run_etag_refreshes(self) {
let mut etag_changes = self.model_service.subscribe_models_etag_changes();
while etag_changes.changed().await.is_ok() {
// 合并同一批并发响应产生的重复通知。
sleep(Duration::from_millis(250)).await;
etag_changes.borrow_and_update();
match self.refresh_once().await {
Ok(result) => info!(
refreshed_plans = result.refreshed_plans,
model_count = result.model_count,
"模型 ETag 变化刷新完成"
),
Err(ModelServiceError::NoAccounts) => {
info!("模型 ETag 变化刷新跳过:没有可用账号");
}
Err(error) => warn!(error = ?error, "模型 ETag 变化刷新失败"),
}
}
}
/// 执行一次模型刷新并同步账号池模型计划约束。
pub async fn refresh_once(&self) -> Result<ModelRefreshResult, ModelServiceError> {
let request_id = uuid::Uuid::new_v4().to_string();
@@ -75,20 +92,20 @@ impl ModelRefreshTask {
.iter()
.map(|selected| ModelRefreshPlanAccount {
plan_type: selected.plan_type.clone(),
account: selected.account.clone(),
access_token: selected.account.access_token.clone(),
account_id: selected.account.account_id.clone(),
installation_id: self
.account_pseudonymizer
.installation_id(&selected.account.id),
})
.collect::<Vec<_>>();
let result = self
.model_service
.refresh_backend_models_with_installation_id(
&plan_accounts,
&request_id,
self.installation_id.as_deref(),
)
.refresh_backend_models(&plan_accounts, &request_id)
.await;
for selected in &selected_accounts {
self.account_pool.release(&selected.account.id).await;
for selected in selected_accounts {
selected.release().await;
}
let result = result?;
@@ -116,6 +133,7 @@ impl PeriodicTaskRunner for ModelRefreshTask {
let mut attempt = 0;
let mut has_fetched_once = false;
let mut deferred_until_accounts_available = false;
while attempt < self.max_retries {
match self.refresh_once().await {
@@ -131,7 +149,8 @@ impl PeriodicTaskRunner for ModelRefreshTask {
Err(error) => {
attempt += 1;
if matches!(error, ModelServiceError::NoAccounts) {
warn!("首次模型刷新跳过:没有可用账户,将进入周期重试");
info!("首次模型刷新跳过:没有可用账户,将进入周期重试");
deferred_until_accounts_available = true;
break;
}
@@ -155,7 +174,7 @@ impl PeriodicTaskRunner for ModelRefreshTask {
}
}
if !has_fetched_once {
if !has_fetched_once && !deferred_until_accounts_available {
warn!("模型刷新任务首次尝试全部失败,切换到周期刷新");
}
@@ -165,8 +184,14 @@ impl PeriodicTaskRunner for ModelRefreshTask {
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
Box::pin(async move {
if let Err(error) = self.refresh_once().await {
warn!(error = ?error, "刷新模型列表失败");
match self.refresh_once().await {
Ok(_) => {}
Err(ModelServiceError::NoAccounts) => {
info!("模型刷新跳过:没有可用账户");
}
Err(error) => {
warn!(error = ?error, "刷新模型列表失败");
}
}
})
}
@@ -6,13 +6,15 @@ use tokio::time::Instant;
use tracing::{debug, info, warn};
use crate::{
upstream::accounts::{
cookies::SqliteCookieStore,
pool::RuntimeAccountPoolService,
quota::{QuotaRefreshSummary, RuntimeQuotaRefreshService},
store::SqliteAccountStore,
fleet::{
cookies::PgCookieStore,
pool::AccountPoolService,
quota::{QuotaRefreshService, QuotaRefreshSummary},
store::PgAccountStore,
},
upstream::transport::CodexBackendClient,
infra::identity::AccountPseudonymizer,
telemetry::account_usage::store::PgAccountUsageStore,
upstream::openai::transport::CodexBackendClient,
};
use super::{
@@ -22,7 +24,7 @@ use super::{
/// 主动配额刷新后台任务。
pub struct QuotaRefreshTask {
service: RuntimeQuotaRefreshService,
service: QuotaRefreshService,
interval_secs: u64,
last_refreshed: HashMap<String, Instant>,
}
@@ -30,15 +32,19 @@ pub struct QuotaRefreshTask {
impl QuotaRefreshTask {
/// 使用自定义间隔构造配额刷新后台任务。
pub fn with_intervals(
store: SqliteAccountStore,
store: PgAccountStore,
usage_store: PgAccountUsageStore,
codex: Arc<CodexBackendClient>,
account_pseudonymizer: Arc<AccountPseudonymizer>,
interval_secs: u64,
min_refresh_interval_secs: u64,
) -> Self {
Self {
service: RuntimeQuotaRefreshService::with_min_refresh_interval_secs(
service: QuotaRefreshService::with_min_refresh_interval_secs(
store,
usage_store,
codex,
account_pseudonymizer,
min_refresh_interval_secs,
),
interval_secs,
@@ -46,20 +52,14 @@ impl QuotaRefreshTask {
}
}
/// 设置 Codex installation id。
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
self.service = self.service.with_installation_id(installation_id);
self
}
/// 设置 usage 请求可复用的账号 Cookie 存储。
pub fn with_cookie_store(mut self, cookie_store: SqliteCookieStore) -> Self {
pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self {
self.service = self.service.with_cookie_store(cookie_store);
self
}
/// 设置运行时账号池,用于刷新后同步内存调度状态。
pub fn with_account_pool(mut self, account_pool: Arc<RuntimeAccountPoolService>) -> Self {
pub fn with_account_pool(mut self, account_pool: Arc<AccountPoolService>) -> Self {
self.service = self.service.with_account_pool(account_pool);
self
}
@@ -89,6 +89,7 @@ impl PeriodicTaskRunner for QuotaRefreshTask {
Ok(summary) => {
debug!(
candidates = summary.candidates,
skipped_cooldown = summary.skipped_cooldown,
skipped_recent = summary.skipped_recent,
failed = summary.failed,
"没有需要刷新的 quota 锁定或待验证账号"
@@ -0,0 +1,91 @@
//! PostgreSQL 增长表保留期清理任务。
use chrono::Utc;
use tracing::{info, warn};
use crate::{
telemetry::{
buckets::store::PgRequestBucketStore, ops::store::PgOpsErrorLogStore,
usage::store::PgUsageRecordStore,
},
upstream::openai::fingerprint::PgFingerprintStore,
};
use super::{
coordinator::SchedulerHandle,
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner, TaskFuture},
};
const RETENTION_TRIM_INTERVAL_SECS: u64 = 60 * 60;
const FINGERPRINT_HISTORY_LIMIT: std::num::NonZeroU32 = std::num::NonZeroU32::new(100).unwrap();
pub struct RetentionTrimTask {
usage_records: PgUsageRecordStore,
ops_errors: PgOpsErrorLogStore,
buckets: PgRequestBucketStore,
fingerprints: PgFingerprintStore,
}
impl RetentionTrimTask {
pub fn new(
usage_records: PgUsageRecordStore,
ops_errors: PgOpsErrorLogStore,
buckets: PgRequestBucketStore,
fingerprints: PgFingerprintStore,
) -> Self {
Self {
usage_records,
ops_errors,
buckets,
fingerprints,
}
}
pub fn start(self) -> SchedulerHandle {
spawn_periodic_task(
self,
PeriodicTaskConfig::new(
RETENTION_TRIM_INTERVAL_SECS,
"PostgreSQL 保留期清理任务已启动",
"PostgreSQL 保留期清理任务已关闭",
),
)
}
/// 立即执行一轮全部 PostgreSQL 保留期清理。
pub async fn run_once(&self) {
let now = Utc::now();
match self.usage_records.trim_to_retention(now).await {
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期成功使用事实"),
Ok(_) => {}
Err(error) => warn!(error = %error, "清理成功使用事实失败"),
}
match self.ops_errors.trim_to_retention(now).await {
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期运维错误事实"),
Ok(_) => {}
Err(error) => warn!(error = %error, "清理运维错误事实失败"),
}
match self.buckets.trim_to_retention(now).await {
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期请求时间桶"),
Ok(_) => {}
Err(error) => warn!(error = %error, "清理请求时间桶失败"),
}
match self
.fingerprints
.trim_update_history_to_limit(FINGERPRINT_HISTORY_LIMIT)
.await
{
Ok(deleted) if deleted > 0 => {
info!(deleted, "已清理超限指纹更新历史")
}
Ok(_) => {}
Err(error) => warn!(error = %error, "清理超限指纹更新历史失败"),
}
}
}
impl PeriodicTaskRunner for RetentionTrimTask {
fn tick(&mut self) -> TaskFuture<'_, ()> {
Box::pin(self.run_once())
}
}
@@ -2,12 +2,11 @@
use tracing::{debug, info, warn};
use crate::upstream::accounts::{
store::SqliteAccountStore,
token_refresh::{
RefreshLeaseStore, RuntimeRefreshPolicy, RuntimeTokenRefreshService, TokenRefresher,
},
use crate::fleet::{
refresh::{RedisRefreshLeaseStore, RuntimeRefreshPolicy, TokenRefreshService},
store::PgAccountStore,
};
use crate::upstream::openai::token_client::TokenRefresher;
use super::{
coordinator::SchedulerHandle,
@@ -21,7 +20,7 @@ pub struct TokenRefreshTask<C>
where
C: TokenRefresher,
{
service: RuntimeTokenRefreshService<C>,
service: TokenRefreshService<C>,
interval_secs: u64,
}
@@ -30,15 +29,15 @@ where
C: TokenRefresher,
{
/// 构造默认后台任务。
pub fn new(store: SqliteAccountStore, policy: RuntimeRefreshPolicy, client: C) -> Self {
pub fn new(store: PgAccountStore, policy: RuntimeRefreshPolicy, client: C) -> Self {
Self {
service: RuntimeTokenRefreshService::new(store, policy, client),
service: TokenRefreshService::new(store, policy, client),
interval_secs: DEFAULT_INTERVAL_SECS,
}
}
/// 使用已创建的运行时刷新服务构造后台任务。
pub fn from_service(service: RuntimeTokenRefreshService<C>) -> Self {
pub fn from_service(service: TokenRefreshService<C>) -> Self {
Self {
service,
interval_secs: DEFAULT_INTERVAL_SECS,
@@ -46,7 +45,7 @@ where
}
/// 使用刷新租约存储保护账号刷新。
pub fn with_refresh_lease_store(mut self, refresh_leases: RefreshLeaseStore) -> Self {
pub fn with_refresh_lease_store(mut self, refresh_leases: RedisRefreshLeaseStore) -> Self {
self.service = self.service.with_refresh_lease_store(refresh_leases);
self
}
@@ -73,7 +72,6 @@ where
info!(
scheduled = summary.scheduled,
immediate = summary.immediate,
recovery_scheduled = summary.recovery_scheduled,
replaced = summary.replaced,
"token 刷新定时器已调度"
);
@@ -94,7 +92,7 @@ where
fn shutdown(&mut self) -> super::periodic::TaskFuture<'_, ()> {
Box::pin(async move {
self.service.clear_scheduled_timers().await;
self.service.shutdown().await;
})
}
}
-31
View File
@@ -1,31 +0,0 @@
use std::{env, path::Path};
use super::schema::AppConfig;
const CONFIG_FILE_ENV: &str = "CPR_CONFIG_FILE";
impl AppConfig {
/// 从 `CPR_CONFIG_FILE` 或当前目录 `config.yaml` 加载配置文件。
pub fn load() -> Result<Self, ::config::ConfigError> {
let config_file = env::var(CONFIG_FILE_ENV)
.ok()
.filter(|value| !value.trim().is_empty());
match config_file {
Some(path) => load_file(path),
None => Self::load_from_dir("."),
}
}
/// 从指定目录加载配置文件。
pub fn load_from_dir(config_dir: impl AsRef<Path>) -> Result<Self, ::config::ConfigError> {
let config_dir = config_dir.as_ref();
load_file(config_dir.join("config.yaml"))
}
}
fn load_file(config_file: impl AsRef<Path>) -> Result<AppConfig, ::config::ConfigError> {
::config::Config::builder()
.add_source(::config::File::from(config_file.as_ref()).required(true))
.build()?
.try_deserialize()
}
-10
View File
@@ -1,10 +0,0 @@
//! 配置加载与配置 schema。
#![deny(missing_docs)]
/// 配置加载器。
pub mod loader;
/// 配置文件 schema 与默认值。
pub mod schema;
/// 管理端可变设置与运行时写回服务。
pub mod settings;
-704
View File
@@ -1,704 +0,0 @@
//! 配置设置领域逻辑与运行时数据库服务。
use std::{
collections::BTreeMap,
sync::{Arc, RwLock as StdRwLock},
};
use chrono::Utc;
use serde::{Deserialize, Serialize};
use sqlx::{sqlite::SqliteRow, Row, SqlitePool};
use subtle::ConstantTimeEq;
use thiserror::Error;
use crate::config::schema::AppConfig;
use crate::infra::identity::generate_admin_api_key;
use crate::upstream::accounts::{
pool::{AccountPoolOptions, RotationStrategy, RuntimeAccountPoolService},
token_refresh::{RefreshPolicy, RuntimeRefreshPolicy},
};
const ROTATION_STRATEGIES: [&str; 3] = ["least_used", "round_robin", "sticky"];
const RUNTIME_SETTINGS_ID: i64 = 1;
/// 管理端可变设置。
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AdminSettings {
/// 模型别名映射。
pub model_aliases: BTreeMap<String, String>,
/// 模型到账号 ID 的显式路由。
pub model_account_routes: BTreeMap<String, Vec<String>>,
/// 访问令牌过期前多少秒开始刷新。
pub refresh_margin_seconds: u64,
/// 访问令牌刷新并发数。
pub refresh_concurrency: u32,
/// 单账号最大并发请求数。
pub max_concurrent_per_account: usize,
/// 同账号请求间隔毫秒数。
pub request_interval_ms: u64,
/// 账号轮换策略。
pub rotation_strategy: String,
}
impl Default for AdminSettings {
fn default() -> Self {
Self {
model_aliases: BTreeMap::new(),
model_account_routes: BTreeMap::new(),
refresh_margin_seconds: 300,
refresh_concurrency: 2,
max_concurrent_per_account: 3,
request_interval_ms: 50,
rotation_strategy: "least_used".to_string(),
}
}
}
/// 管理端设置补丁。
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminSettingsPatch {
/// 模型别名映射。
pub model_aliases: Option<BTreeMap<String, String>>,
/// 模型到账号 ID 的显式路由。
pub model_account_routes: Option<BTreeMap<String, Vec<String>>>,
/// 访问令牌过期前多少秒开始刷新。
pub refresh_margin_seconds: Option<u64>,
/// 访问令牌刷新并发数。
pub refresh_concurrency: Option<u32>,
/// 单账号最大并发请求数。
pub max_concurrent_per_account: Option<usize>,
/// 同账号请求间隔毫秒数。
pub request_interval_ms: Option<u64>,
/// 账号轮换策略。
pub rotation_strategy: Option<String>,
}
/// 设置领域服务。
#[derive(Debug, Clone, Default)]
pub struct SettingsService;
impl SettingsService {
/// 将管理端设置补丁应用到当前设置。
pub fn apply_patch(
current: &mut AdminSettings,
patch: AdminSettingsPatch,
) -> Result<(), SettingsServiceError> {
if let Some(model_aliases) = patch.model_aliases {
current.model_aliases = validate_model_aliases(model_aliases)?;
}
if let Some(model_account_routes) = patch.model_account_routes {
current.model_account_routes = validate_model_account_routes(model_account_routes)?;
}
if let Some(refresh_margin_seconds) = patch.refresh_margin_seconds {
current.refresh_margin_seconds =
positive_u64("refreshMarginSeconds", refresh_margin_seconds)?;
}
if let Some(refresh_concurrency) = patch.refresh_concurrency {
current.refresh_concurrency = positive_u32("refreshConcurrency", refresh_concurrency)?;
}
if let Some(max_concurrent_per_account) = patch.max_concurrent_per_account {
current.max_concurrent_per_account =
positive_usize("maxConcurrentPerAccount", max_concurrent_per_account)?;
}
if let Some(request_interval_ms) = patch.request_interval_ms {
current.request_interval_ms = request_interval_ms;
}
if let Some(rotation_strategy) = patch.rotation_strategy {
current.rotation_strategy = validate_rotation_strategy(&rotation_strategy)?;
}
Ok(())
}
}
/// 设置领域错误。
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum SettingsServiceError {
/// 字段值无效。
#[error("invalid setting `{field}`: {message}")]
InvalidField {
/// 字段名。
field: String,
/// 错误说明。
message: String,
},
}
impl SettingsServiceError {
/// 返回无效字段名。
pub fn field(&self) -> &str {
match self {
Self::InvalidField { field, .. } => field,
}
}
}
fn validate_model_aliases(
aliases: BTreeMap<String, String>,
) -> Result<BTreeMap<String, String>, SettingsServiceError> {
let mut normalized = BTreeMap::new();
for (alias, target) in aliases {
let alias = non_empty_string("modelAliases", &alias)?;
let target = non_empty_string("modelAliases", &target)?;
if alias == target {
return Err(invalid_field(
"modelAliases",
"alias and target must differ",
));
}
normalized.insert(alias, target);
}
Ok(normalized)
}
fn validate_model_account_routes(
routes: BTreeMap<String, Vec<String>>,
) -> Result<BTreeMap<String, Vec<String>>, SettingsServiceError> {
let mut normalized = BTreeMap::new();
for (model, account_ids) in routes {
let model = non_empty_string("modelAccountRoutes", &model)?;
let mut normalized_account_ids = Vec::new();
for account_id in account_ids {
let account_id = non_empty_string("modelAccountRoutes", &account_id)?;
if normalized_account_ids.contains(&account_id) {
return Err(invalid_field(
"modelAccountRoutes",
format!("duplicate account id `{account_id}`"),
));
}
normalized_account_ids.push(account_id);
}
if normalized_account_ids.is_empty() {
return Err(invalid_field(
"modelAccountRoutes",
"account list must not be empty",
));
}
normalized.insert(model, normalized_account_ids);
}
Ok(normalized)
}
fn validate_rotation_strategy(strategy: &str) -> Result<String, SettingsServiceError> {
let strategy = non_empty_string("rotationStrategy", strategy)?;
if ROTATION_STRATEGIES.contains(&strategy.as_str()) {
Ok(strategy)
} else {
Err(invalid_field(
"rotationStrategy",
"must be one of least_used, round_robin, sticky",
))
}
}
fn non_empty_string(field: &str, value: &str) -> Result<String, SettingsServiceError> {
let value = value.trim().to_string();
if value.is_empty() {
Err(invalid_field(field, "must not be empty"))
} else {
Ok(value)
}
}
fn positive_u64(field: &str, value: u64) -> Result<u64, SettingsServiceError> {
if value == 0 {
Err(invalid_field(field, "must be greater than 0"))
} else {
Ok(value)
}
}
fn positive_u32(field: &str, value: u32) -> Result<u32, SettingsServiceError> {
if value == 0 {
Err(invalid_field(field, "must be greater than 0"))
} else {
Ok(value)
}
}
fn positive_usize(field: &str, value: usize) -> Result<usize, SettingsServiceError> {
if value == 0 {
Err(invalid_field(field, "must be greater than 0"))
} else {
Ok(value)
}
}
fn invalid_field(field: &str, message: impl Into<String>) -> SettingsServiceError {
SettingsServiceError::InvalidField {
field: field.to_string(),
message: message.into(),
}
}
/// 管理员 API Key 状态。
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminApiKeyStatus {
/// 是否已经生成。
pub exists: bool,
/// 脱敏后的密钥。
pub masked_key: Option<String>,
}
/// 运行时设置服务。
#[derive(Clone)]
pub struct RuntimeSettingsService {
current: Arc<StdRwLock<Arc<AppConfig>>>,
pool: SqlitePool,
account_pool: Arc<RuntimeAccountPoolService>,
refresh_policy: RuntimeRefreshPolicy,
}
impl RuntimeSettingsService {
/// 构造运行时设置服务。
pub fn new(
config: AppConfig,
pool: SqlitePool,
account_pool: Arc<RuntimeAccountPoolService>,
refresh_policy: RuntimeRefreshPolicy,
) -> Self {
Self {
current: Arc::new(StdRwLock::new(Arc::new(config))),
pool,
account_pool,
refresh_policy,
}
}
/// 返回当前运行时配置快照。
pub fn current(&self) -> Arc<AppConfig> {
self.current
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
/// 初始化缺省运行设置,并返回数据库中的运行配置快照。
pub async fn load_or_initialize_config(
mut config: AppConfig,
pool: &SqlitePool,
) -> Result<AppConfig, RuntimeSettingsError> {
let settings = admin_settings_from_config(&config);
insert_runtime_settings_if_missing(pool, &settings).await?;
let settings = load_runtime_settings(pool).await?;
apply_admin_settings_to_config(&mut config, settings);
Ok(config)
}
/// 应用设置补丁、写入数据库并更新运行时配置快照。
pub async fn update(
&self,
patch: AdminSettingsPatch,
) -> Result<Arc<AppConfig>, RuntimeSettingsError> {
let mut next = (*self.current()).clone();
let mut settings = admin_settings_from_config(&next);
SettingsService::apply_patch(&mut settings, patch)?;
save_runtime_settings(&self.pool, &settings).await?;
apply_admin_settings_to_config(&mut next, settings);
self.account_pool
.apply_options(
account_pool_options_from_config(&next),
next.auth.request_interval_ms,
)
.await;
self.refresh_policy.update(RefreshPolicy {
refresh_margin_seconds: next.auth.refresh_margin_seconds,
refresh_concurrency: next.auth.refresh_concurrency,
});
let next = Arc::new(next);
*self
.current
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = next.clone();
Ok(next)
}
/// 返回管理员 API Key 状态。
pub async fn admin_api_key_status(&self) -> Result<AdminApiKeyStatus, RuntimeSettingsError> {
admin_api_key_status(&self.pool).await
}
/// 重新生成管理员 API Key,完整值只在本次返回。
pub async fn regenerate_admin_api_key(&self) -> Result<String, RuntimeSettingsError> {
let settings = admin_settings_from_config(&self.current());
insert_runtime_settings_if_missing(&self.pool, &settings).await?;
let key = generate_admin_api_key();
sqlx::query(
r"
update runtime_settings
set admin_api_key = ?, updated_at = ?
where id = ?",
)
.bind(&key)
.bind(Utc::now().to_rfc3339())
.bind(RUNTIME_SETTINGS_ID)
.execute(&self.pool)
.await?;
Ok(key)
}
/// 删除管理员 API Key。
pub async fn delete_admin_api_key(&self) -> Result<(), RuntimeSettingsError> {
sqlx::query(
r"
update runtime_settings
set admin_api_key = null, updated_at = ?
where id = ?",
)
.bind(Utc::now().to_rfc3339())
.bind(RUNTIME_SETTINGS_ID)
.execute(&self.pool)
.await?;
Ok(())
}
/// 校验管理员 API Key。
pub async fn verify_admin_api_key(&self, key: &str) -> Result<bool, RuntimeSettingsError> {
if key.is_empty() {
return Ok(false);
}
let stored = load_admin_api_key(&self.pool).await?;
Ok(stored
.as_deref()
.filter(|stored_key| !stored_key.is_empty())
.is_some_and(|stored_key| key.as_bytes().ct_eq(stored_key.as_bytes()).into()))
}
}
/// 运行时设置错误。
#[derive(Debug, Error)]
pub enum RuntimeSettingsError {
/// 设置字段校验失败。
#[error(transparent)]
InvalidField(#[from] SettingsServiceError),
/// 数据库操作失败。
#[error("runtime settings database error: {0}")]
Database(#[from] sqlx::Error),
/// JSON 编解码失败。
#[error("runtime settings json error: {0}")]
Json(#[from] serde_json::Error),
/// 数据库存储值非法。
#[error("invalid stored setting `{field}`: {message}")]
StoredField {
/// 字段名。
field: String,
/// 错误说明。
message: String,
},
}
fn admin_settings_from_config(config: &AppConfig) -> AdminSettings {
AdminSettings {
model_aliases: config.model_aliases.clone(),
model_account_routes: config.model_account_routes.clone(),
refresh_margin_seconds: config.auth.refresh_margin_seconds,
refresh_concurrency: config.auth.refresh_concurrency,
max_concurrent_per_account: config.auth.max_concurrent_per_account,
request_interval_ms: config.auth.request_interval_ms,
rotation_strategy: config.auth.rotation_strategy.clone(),
}
}
fn apply_admin_settings_to_config(config: &mut AppConfig, settings: AdminSettings) {
config.model_aliases = settings.model_aliases;
config.model_account_routes = settings.model_account_routes;
config.auth.refresh_margin_seconds = settings.refresh_margin_seconds;
config.auth.refresh_concurrency = settings.refresh_concurrency;
config.auth.max_concurrent_per_account = settings.max_concurrent_per_account;
config.auth.request_interval_ms = settings.request_interval_ms;
config.auth.rotation_strategy = settings.rotation_strategy;
}
async fn insert_runtime_settings_if_missing(
pool: &SqlitePool,
settings: &AdminSettings,
) -> Result<(), RuntimeSettingsError> {
let model_aliases_json = serde_json::to_string(&settings.model_aliases)?;
let mut tx = pool.begin().await?;
let result = sqlx::query(
r"
insert or ignore into runtime_settings (
id,
model_aliases_json,
refresh_margin_seconds,
refresh_concurrency,
max_concurrent_per_account,
request_interval_ms,
rotation_strategy,
updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?)",
)
.bind(RUNTIME_SETTINGS_ID)
.bind(model_aliases_json)
.bind(
i64::try_from(settings.refresh_margin_seconds)
.map_err(|_| stored_field_error("refreshMarginSeconds", "out of range"))?,
)
.bind(i64::from(settings.refresh_concurrency))
.bind(
i64::try_from(settings.max_concurrent_per_account)
.map_err(|_| stored_field_error("maxConcurrentPerAccount", "out of range"))?,
)
.bind(
i64::try_from(settings.request_interval_ms)
.map_err(|_| stored_field_error("requestIntervalMs", "out of range"))?,
)
.bind(&settings.rotation_strategy)
.bind(Utc::now().to_rfc3339())
.execute(&mut *tx)
.await?;
if result.rows_affected() > 0 {
replace_model_account_routes(&mut tx, &settings.model_account_routes).await?;
}
tx.commit().await?;
Ok(())
}
async fn save_runtime_settings(
pool: &SqlitePool,
settings: &AdminSettings,
) -> Result<(), RuntimeSettingsError> {
let model_aliases_json = serde_json::to_string(&settings.model_aliases)?;
let mut tx = pool.begin().await?;
sqlx::query(
r"
insert into runtime_settings (
id,
model_aliases_json,
refresh_margin_seconds,
refresh_concurrency,
max_concurrent_per_account,
request_interval_ms,
rotation_strategy,
updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?)
on conflict(id) do update set
model_aliases_json = excluded.model_aliases_json,
refresh_margin_seconds = excluded.refresh_margin_seconds,
refresh_concurrency = excluded.refresh_concurrency,
max_concurrent_per_account = excluded.max_concurrent_per_account,
request_interval_ms = excluded.request_interval_ms,
rotation_strategy = excluded.rotation_strategy,
updated_at = excluded.updated_at",
)
.bind(RUNTIME_SETTINGS_ID)
.bind(model_aliases_json)
.bind(
i64::try_from(settings.refresh_margin_seconds)
.map_err(|_| stored_field_error("refreshMarginSeconds", "out of range"))?,
)
.bind(i64::from(settings.refresh_concurrency))
.bind(
i64::try_from(settings.max_concurrent_per_account)
.map_err(|_| stored_field_error("maxConcurrentPerAccount", "out of range"))?,
)
.bind(
i64::try_from(settings.request_interval_ms)
.map_err(|_| stored_field_error("requestIntervalMs", "out of range"))?,
)
.bind(&settings.rotation_strategy)
.bind(Utc::now().to_rfc3339())
.execute(&mut *tx)
.await?;
replace_model_account_routes(&mut tx, &settings.model_account_routes).await?;
tx.commit().await?;
Ok(())
}
async fn load_runtime_settings(pool: &SqlitePool) -> Result<AdminSettings, RuntimeSettingsError> {
let row = sqlx::query(
r"
select
model_aliases_json,
refresh_margin_seconds,
refresh_concurrency,
max_concurrent_per_account,
request_interval_ms,
rotation_strategy
from runtime_settings
where id = ?",
)
.bind(RUNTIME_SETTINGS_ID)
.fetch_one(pool)
.await?;
let mut settings = runtime_settings_from_row(&row)?;
settings.model_account_routes = load_model_account_routes(pool).await?;
Ok(settings)
}
async fn admin_api_key_status(
pool: &SqlitePool,
) -> Result<AdminApiKeyStatus, RuntimeSettingsError> {
let key = load_admin_api_key(pool).await?;
let masked_key = key
.as_deref()
.filter(|key| !key.is_empty())
.map(mask_api_key);
Ok(AdminApiKeyStatus {
exists: masked_key.is_some(),
masked_key,
})
}
async fn load_admin_api_key(pool: &SqlitePool) -> Result<Option<String>, RuntimeSettingsError> {
Ok(sqlx::query_scalar::<_, Option<String>>(
"select admin_api_key from runtime_settings where id = ?",
)
.bind(RUNTIME_SETTINGS_ID)
.fetch_optional(pool)
.await?
.flatten())
}
fn mask_api_key(key: &str) -> String {
if key.len() > 14 {
format!("{}...{}", &key[..10], &key[key.len() - 4..])
} else {
key.to_string()
}
}
fn runtime_settings_from_row(row: &SqliteRow) -> Result<AdminSettings, RuntimeSettingsError> {
let model_aliases_json: String = row.get("model_aliases_json");
Ok(AdminSettings {
model_aliases: serde_json::from_str(&model_aliases_json)?,
model_account_routes: BTreeMap::new(),
refresh_margin_seconds: positive_i64_to_u64(
"refreshMarginSeconds",
row.get("refresh_margin_seconds"),
)?,
refresh_concurrency: positive_i64_to_u32(
"refreshConcurrency",
row.get("refresh_concurrency"),
)?,
max_concurrent_per_account: positive_i64_to_usize(
"maxConcurrentPerAccount",
row.get("max_concurrent_per_account"),
)?,
request_interval_ms: non_negative_i64_to_u64(
"requestIntervalMs",
row.get("request_interval_ms"),
)?,
rotation_strategy: validate_rotation_strategy(&row.get::<String, _>("rotation_strategy"))?,
})
}
async fn replace_model_account_routes(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
routes: &BTreeMap<String, Vec<String>>,
) -> Result<(), RuntimeSettingsError> {
sqlx::query("delete from model_account_routes")
.execute(&mut **tx)
.await?;
let now = Utc::now().to_rfc3339();
for (model, account_ids) in routes {
for (priority, account_id) in account_ids.iter().enumerate() {
sqlx::query(
r"
insert into model_account_routes (
model,
account_id,
priority,
enabled,
created_at,
updated_at
) values (?, ?, ?, 1, ?, ?)",
)
.bind(model)
.bind(account_id)
.bind(
i64::try_from(priority)
.map_err(|_| stored_field_error("modelAccountRoutes", "out of range"))?,
)
.bind(&now)
.bind(&now)
.execute(&mut **tx)
.await?;
}
}
Ok(())
}
async fn load_model_account_routes(
pool: &SqlitePool,
) -> Result<BTreeMap<String, Vec<String>>, RuntimeSettingsError> {
let rows: Vec<(String, String)> = sqlx::query_as(
r"
select model, account_id
from model_account_routes
where enabled = 1
order by model asc, priority asc, account_id asc",
)
.fetch_all(pool)
.await?;
let mut routes = BTreeMap::<String, Vec<String>>::new();
for (model, account_id) in rows {
routes.entry(model).or_default().push(account_id);
}
Ok(routes)
}
/// 从当前运行配置生成账号池调度参数。
pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions {
AccountPoolOptions {
max_concurrent_per_account: config.auth.max_concurrent_per_account,
rotation_strategy: rotation_strategy_from_config(&config.auth.rotation_strategy),
skip_quota_limited: config.quota.skip_exhausted,
tier_priority: config.auth.tier_priority.clone(),
model_account_routes: config.model_account_routes.clone(),
..AccountPoolOptions::default()
}
}
fn rotation_strategy_from_config(strategy: &str) -> RotationStrategy {
match strategy {
"round_robin" => RotationStrategy::RoundRobin,
"sticky" => RotationStrategy::Sticky,
_ => RotationStrategy::LeastUsed,
}
}
fn positive_i64_to_u64(field: &'static str, value: i64) -> Result<u64, RuntimeSettingsError> {
if value <= 0 {
return Err(stored_field_error(field, "must be greater than 0"));
}
u64::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
}
fn positive_i64_to_u32(field: &'static str, value: i64) -> Result<u32, RuntimeSettingsError> {
if value <= 0 {
return Err(stored_field_error(field, "must be greater than 0"));
}
u32::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
}
fn positive_i64_to_usize(field: &'static str, value: i64) -> Result<usize, RuntimeSettingsError> {
if value <= 0 {
return Err(stored_field_error(field, "must be greater than 0"));
}
usize::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
}
fn non_negative_i64_to_u64(field: &'static str, value: i64) -> Result<u64, RuntimeSettingsError> {
if value < 0 {
return Err(stored_field_error(
field,
"must be greater than or equal to 0",
));
}
u64::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
}
fn stored_field_error(
field: impl Into<String>,
message: impl Into<String>,
) -> RuntimeSettingsError {
RuntimeSettingsError::StoredField {
field: field.into(),
message: message.into(),
}
}
+171
View File
@@ -0,0 +1,171 @@
//! 客户端身份、本地会话索引与账号作用域 wire 身份隔离。
use crate::{
infra::identity::AccountPseudonymizer,
upstream::openai::protocol::responses::{CodexCompactRequest, CodexResponsesRequest},
};
/// 持久化密钥驱动的账号身份隔离服务。
#[derive(Clone)]
pub struct AccountIdentityService {
pseudonymizer: AccountPseudonymizer,
}
impl std::fmt::Debug for AccountIdentityService {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("AccountIdentityService")
.finish_non_exhaustive()
}
}
/// 单次账号 attempt 的 wire 身份。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountScopedIdentity {
pub prompt_cache_key: Option<String>,
pub session_id: Option<String>,
pub thread_id: Option<String>,
pub client_request_id: String,
pub turn_id: Option<String>,
pub window_id: Option<String>,
pub parent_thread_id: Option<String>,
pub installation_id: String,
}
struct IdentityValues<'a> {
prompt_cache_key: Option<&'a str>,
session_id: Option<&'a str>,
thread_id: Option<&'a str>,
client_request_id: Option<&'a str>,
turn_id: Option<&'a str>,
window_id: Option<&'a str>,
parent_thread_id: Option<&'a str>,
}
impl AccountIdentityService {
pub fn new(pseudonymizer: AccountPseudonymizer) -> Self {
Self { pseudonymizer }
}
pub fn prepare_local_identity(&self, request: &mut CodexResponsesRequest) {
if request.local_conversation_id.is_some() {
return;
}
let anchor = request
.prompt_cache_key()
.or(request.client_session_id.as_deref())
.or(request.client_thread_id.as_deref())
.or(request.client_conversation_id.as_deref())
.map(ToString::to_string)
.or_else(|| super::resolve::derive_stable_conversation_key(request));
request.local_conversation_id = anchor
.as_deref()
.map(|anchor| format!("lc_{}", self.pseudonym("local-conversation", None, anchor)));
}
pub fn scope(
&self,
request: &CodexResponsesRequest,
account_id: &str,
proxy_request_id: &str,
) -> AccountScopedIdentity {
self.scope_values(
account_id,
proxy_request_id,
IdentityValues {
prompt_cache_key: request.prompt_cache_key(),
session_id: request.client_session_id.as_deref(),
thread_id: request.client_thread_id.as_deref(),
client_request_id: request.client_request_id.as_deref(),
turn_id: request.client_turn_id.as_deref(),
window_id: request.codex_window_id.as_deref(),
parent_thread_id: request.parent_thread_id.as_deref(),
},
)
}
pub fn scope_compact(
&self,
request: &CodexCompactRequest,
account_id: &str,
proxy_request_id: &str,
) -> AccountScopedIdentity {
self.scope_values(
account_id,
proxy_request_id,
IdentityValues {
prompt_cache_key: request.prompt_cache_key(),
session_id: request.client_session_id.as_deref(),
thread_id: request.client_thread_id.as_deref(),
client_request_id: request.client_request_id.as_deref(),
turn_id: request.client_turn_id.as_deref(),
window_id: request.client_window_id.as_deref(),
parent_thread_id: request.client_parent_thread_id.as_deref(),
},
)
}
pub fn scope_auxiliary(
&self,
account_id: &str,
proxy_request_id: &str,
) -> AccountScopedIdentity {
AccountScopedIdentity {
client_request_id: format!(
"cr_{}",
self.pseudonym("client-request-id", Some(account_id), proxy_request_id)
),
installation_id: self.pseudonymizer.installation_id(account_id),
..AccountScopedIdentity::default()
}
}
fn scope_values(
&self,
account_id: &str,
proxy_request_id: &str,
values: IdentityValues<'_>,
) -> AccountScopedIdentity {
AccountScopedIdentity {
prompt_cache_key: self.scope_optional(
"prompt-cache-key",
account_id,
values.prompt_cache_key,
),
session_id: self.scope_optional("session-id", account_id, values.session_id),
thread_id: self.scope_optional("thread-id", account_id, values.thread_id),
client_request_id: format!(
"cr_{}",
self.pseudonym(
"client-request-id",
Some(account_id),
values.client_request_id.unwrap_or(proxy_request_id),
)
),
turn_id: self.scope_optional("turn-id", account_id, values.turn_id),
window_id: self.scope_optional("window-id", account_id, values.window_id),
parent_thread_id: self.scope_optional(
"parent-thread-id",
account_id,
values.parent_thread_id,
),
installation_id: self.pseudonymizer.installation_id(account_id),
}
}
fn scope_optional(
&self,
domain: &str,
account_id: &str,
value: Option<&str>,
) -> Option<String> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| format!("wi_{}", self.pseudonym(domain, Some(account_id), value)))
}
fn pseudonym(&self, domain: &str, account_id: Option<&str>, value: &str) -> String {
self.pseudonymizer.scoped(domain, account_id, value)
}
}
+13
View File
@@ -0,0 +1,13 @@
//! 会话亲和身份、存储、解析与运行时服务。
pub mod identity;
pub mod resolve;
pub mod service;
pub mod store;
pub mod types;
pub use identity::{AccountIdentityService, AccountScopedIdentity};
pub use resolve::*;
pub use service::*;
pub use store::*;
pub use types::*;
+248
View File
@@ -0,0 +1,248 @@
use chrono::{DateTime, Utc};
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::{
dispatch::{
affinity::{SessionAffinityEntry, SessionAffinityService},
recovery::history::HistoryRecoveryPlan,
service::ResponseDispatchService,
},
upstream::openai::protocol::{
events::TokenUsage,
responses::{completed_response_metadata, CodexResponsesRequest},
},
};
impl ResponseDispatchService {
pub(in crate::dispatch) async fn preferred_account_id_for_request(
&self,
request: &CodexResponsesRequest,
history: &HistoryRecoveryPlan,
now: DateTime<Utc>,
) -> Option<String> {
if let Some(account_id) = history.preferred_account_id() {
return Some(account_id.to_string());
}
if request.previous_response_id().is_some() {
return None;
}
self.conversation_account_id(request, now).await
}
async fn conversation_account_id(
&self,
request: &CodexResponsesRequest,
now: DateTime<Utc>,
) -> Option<String> {
let conversation_id = request
.local_conversation_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())?;
let variant_hash = compute_variant_hash(request);
self.session_affinity
.lookup_latest_account_by_conversation(conversation_id, None, Some(&variant_hash), now)
.await
}
}
pub(in crate::dispatch) async fn record_response_affinity(
session_affinity: &SessionAffinityService,
history: &HistoryRecoveryPlan,
original_request: &CodexResponsesRequest,
account_id: &str,
body: &str,
turn_state: Option<String>,
usage: Option<TokenUsage>,
) {
let metadata = match completed_response_metadata(body) {
Ok(Some(metadata)) => metadata,
Ok(None) => return,
Err(error) => {
tracing::warn!(
error = %error,
"failed to parse completed response metadata for session affinity"
);
return;
}
};
let conversation_id = history
.conversation_id(original_request)
.unwrap_or(&metadata.response_id)
.to_string();
let variant_hash = compute_variant_hash(original_request);
let entry = SessionAffinityEntry {
account_id: account_id.to_string(),
conversation_id,
turn_state: turn_state
.filter(|value| !value.trim().is_empty())
.or_else(|| original_request.turn_state.clone()),
instructions_hash: Some(hash_instructions(Some(original_request.instructions()))),
input_tokens: usage.map(|usage| usage.input_tokens),
function_call_ids: metadata.function_call_ids,
variant_hash: Some(variant_hash),
continuation_scope: if original_request.store() {
crate::upstream::openai::protocol::responses::PreviousResponseScope::Persisted
} else {
crate::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal
},
replay: history.completed_replay(original_request, &metadata.output),
created_at: Utc::now(),
};
if let Err(error) = session_affinity
.record(metadata.response_id.clone(), entry)
.await
{
tracing::warn!(
error = %error,
response_id = %metadata.response_id,
account_id = %account_id,
"failed to record session affinity"
);
}
}
// ====================================================================
// Variant identity helpers
// ====================================================================
/// 准备用于区分并发分支的变体身份。
pub fn prepare_variant_identity(request: &mut CodexResponsesRequest) {
request.variant_identity = build_variant_identity(request);
}
/// 计算请求变体哈希。
pub fn compute_variant_hash(request: &CodexResponsesRequest) -> String {
compute_variant_hash_with_identity(request, request.variant_identity.as_deref())
}
fn compute_variant_hash_with_identity(
request: &CodexResponsesRequest,
identity: Option<&str>,
) -> String {
let tools_json = request.tools().map_or_else(|| "[]".to_string(), tools_json);
let mut hasher = Sha256::new();
hasher.update(request.instructions().as_bytes());
hasher.update(b"\0");
hasher.update(tools_json.as_bytes());
if let Some(identity) = identity
.map(str::trim)
.filter(|identity| !identity.is_empty())
{
hasher.update(b"\0");
hasher.update(identity.as_bytes());
}
hex::encode(hasher.finalize()).chars().take(12).collect()
}
fn tools_json(tools: &[Value]) -> String {
serde_json::to_string(tools).unwrap_or_else(|_| "[]".to_string())
}
fn build_variant_identity(request: &CodexResponsesRequest) -> Option<String> {
let mut parts = Vec::with_capacity(2);
if let Some(window_id) = non_empty_str(request.codex_window_id.as_deref()) {
parts.push(format!("window:{window_id}"));
}
if request.explicit_prompt_cache_key
|| non_empty_str(request.client_conversation_id.as_deref()).is_some()
{
if let Some(anchor) = derive_stable_conversation_key(request) {
parts.push(format!("anchor:{anchor}"));
}
}
(!parts.is_empty()).then(|| parts.join("\0"))
}
fn non_empty_str(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
/// 计算 instructions 哈希。
pub fn hash_instructions(instructions: Option<&str>) -> String {
let mut hasher = Sha256::new();
hasher.update(instructions.unwrap_or_default().as_bytes());
hex::encode(hasher.finalize())
}
const LEADING_SYSTEM_REMINDER_OPEN: &str = "<system-reminder>";
const LEADING_SYSTEM_REMINDER_CLOSE: &str = "</system-reminder>";
/// 按原版 `stable-conversation-key.ts` 的规则派生稳定 conversation key。
pub fn derive_stable_conversation_key(request: &CodexResponsesRequest) -> Option<String> {
let instructions = request
.instructions()
.chars()
.take(2000)
.collect::<String>();
let first_user_text = first_user_text(request.input());
let normalized_first_user_text = normalize_conversation_anchor_text(&first_user_text);
let first_user_text = if normalized_first_user_text.is_empty() {
first_user_text
} else {
normalized_first_user_text
};
if instructions.is_empty() && first_user_text.is_empty() {
return None;
}
let mut hasher = Sha256::new();
hasher.update(request.model().as_bytes());
hasher.update(b"\0");
hasher.update(instructions.as_bytes());
hasher.update(b"\0");
hasher.update(first_user_text.as_bytes());
let hex = hex::encode(hasher.finalize());
Some(format!(
"{}-{}-{}-{}-{}",
&hex[0..8],
&hex[8..12],
&hex[12..16],
&hex[16..20],
&hex[20..32]
))
}
fn first_user_text(input: &[Value]) -> String {
for item in input {
if item.get("role").and_then(Value::as_str) != Some("user") {
continue;
}
let Some(content) = item.get("content") else {
return String::new();
};
if let Some(text) = content.as_str() {
return text.to_string();
}
if let Some(parts) = content.as_array() {
return parts
.iter()
.filter(|part| part.get("type").and_then(Value::as_str) == Some("input_text"))
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect::<String>();
}
return String::new();
}
String::new()
}
fn normalize_conversation_anchor_text(text: &str) -> String {
let mut rest = text.trim_start();
loop {
let lower = rest.to_ascii_lowercase();
if !lower.starts_with(LEADING_SYSTEM_REMINDER_OPEN) {
break;
}
let Some(close_start) = lower.find(LEADING_SYSTEM_REMINDER_CLOSE) else {
break;
};
let close_end = close_start + LEADING_SYSTEM_REMINDER_CLOSE.len();
rest = rest[close_end..].trim_start();
}
rest.to_string()
}
+147
View File
@@ -0,0 +1,147 @@
//! 运行时会话亲和服务。
use chrono::{DateTime, Duration, Utc};
use thiserror::Error;
use super::{
store::{RedisSessionAffinityStore, RedisSessionAffinityStoreError},
types::SessionAffinityEntry,
};
const DEFAULT_SESSION_AFFINITY_TTL_SECS: i64 = 4 * 60 * 60;
pub const MAX_REPLAY_SNAPSHOT_BYTES: u64 = 2 * 1024 * 1024;
pub const MAX_REPLAY_SESSION_BYTES: u64 = 16 * 1024 * 1024;
pub const MAX_REPLAY_DEPTH: u16 = 128;
/// 运行时会话亲和性服务。
#[derive(Clone)]
pub struct SessionAffinityService {
store: RedisSessionAffinityStore,
ttl: Duration,
}
/// 运行时会话亲和性错误。
#[derive(Debug, Error)]
pub enum SessionAffinityError {
#[error("session affinity store error: {0}")]
Store(#[from] RedisSessionAffinityStoreError),
}
impl SessionAffinityService {
pub fn new(store: RedisSessionAffinityStore) -> Self {
let ttl = Duration::seconds(DEFAULT_SESSION_AFFINITY_TTL_SECS);
Self { store, ttl }
}
pub async fn record(
&self,
response_id: String,
entry: SessionAffinityEntry,
) -> Result<(), SessionAffinityError> {
Ok(self.store.upsert(&response_id, &entry, self.ttl).await?)
}
pub async fn lookup(
&self,
response_id: &str,
now: DateTime<Utc>,
) -> Option<SessionAffinityEntry> {
self.entry(response_id, now).await
}
pub async fn replay_input(
&self,
response_id: &str,
entry: &SessionAffinityEntry,
now: DateTime<Utc>,
) -> Option<Vec<serde_json::Value>> {
match self
.store
.replay_input(
response_id,
entry,
now,
self.ttl,
MAX_REPLAY_DEPTH,
MAX_REPLAY_SESSION_BYTES,
)
.await
{
Ok(input) => input,
Err(error) => {
tracing::warn!(
response_id,
error = %error,
"failed to reconstruct Redis response replay history"
);
None
}
}
}
pub async fn lookup_latest_account_by_conversation(
&self,
conversation_id: &str,
max_age: Option<Duration>,
variant_hash: Option<&str>,
now: DateTime<Utc>,
) -> Option<String> {
self.latest(conversation_id, max_age, variant_hash, now)
.await
.map(|(_, entry)| entry.account_id)
}
pub async fn forget(&self, response_id: &str) -> bool {
match self.store.forget(response_id).await {
Ok(forgotten) => forgotten,
Err(error) => {
tracing::warn!(
response_id,
error = %error,
"failed to remove Redis session affinity"
);
false
}
}
}
pub async fn forget_account(&self, account_id: &str) -> bool {
match self.store.forget_account(account_id).await {
Ok(forgotten) => forgotten > 0,
Err(error) => {
tracing::warn!(account_id, error = %error, "failed to remove account affinities");
false
}
}
}
async fn entry(&self, response_id: &str, now: DateTime<Utc>) -> Option<SessionAffinityEntry> {
match self.store.get(response_id, now, self.ttl).await {
Ok(entry) => entry,
Err(error) => {
tracing::warn!(response_id, error = %error, "failed to read Redis session affinity");
None
}
}
}
async fn latest(
&self,
conversation_id: &str,
max_age: Option<Duration>,
variant_hash: Option<&str>,
now: DateTime<Utc>,
) -> Option<(String, SessionAffinityEntry)> {
match self
.store
.latest_by_conversation(conversation_id, max_age, variant_hash, now, self.ttl)
.await
{
Ok(entry) => entry,
Err(error) => {
tracing::warn!(conversation_id, error = %error, "failed to query Redis affinity index");
None
}
}
}
}
+358
View File
@@ -0,0 +1,358 @@
//! Redis 会话亲和存储。
use std::collections::HashMap;
use chrono::{DateTime, Duration, Utc};
use redis::AsyncCommands;
use serde_json::Value;
use thiserror::Error;
use crate::infra::redis::RedisConnection;
use super::types::SessionAffinityEntry;
/// Redis 会话亲和性存储。
#[derive(Clone)]
pub struct RedisSessionAffinityStore {
redis: RedisConnection,
}
/// Redis 会话亲和性存储错误。
#[derive(Debug, Error)]
pub enum RedisSessionAffinityStoreError {
#[error("Redis session affinity operation failed: {0}")]
Redis(#[from] redis::RedisError),
#[error("invalid Redis session affinity value: {0}")]
Json(#[from] serde_json::Error),
}
pub type RedisSessionAffinityStoreResult<T> = Result<T, RedisSessionAffinityStoreError>;
impl RedisSessionAffinityStore {
pub fn new(redis: RedisConnection) -> Self {
Self { redis }
}
pub async fn upsert(
&self,
response_id: &str,
entry: &SessionAffinityEntry,
ttl: Duration,
) -> RedisSessionAffinityStoreResult<()> {
let value = serde_json::to_string(entry)?;
let expires_at = entry
.created_at
.checked_add_signed(ttl)
.unwrap_or(DateTime::<Utc>::MAX_UTC);
if expires_at <= Utc::now() {
return Ok(());
}
let response_key = self.response_key(response_id);
let conversation_key = self.conversation_key(&entry.conversation_id);
let account_key = self.account_key(&entry.account_id);
let ttl_seconds = ttl.num_seconds().max(1);
let cutoff_ms = Utc::now().timestamp_millis() - ttl.num_milliseconds();
let mut connection = self.redis.manager();
let mut transaction = redis::pipe();
transaction
.atomic()
.cmd("SET")
.arg(response_key)
.arg(value)
.arg("EXAT")
.arg(expires_at.timestamp())
.ignore()
.cmd("ZADD")
.arg(&conversation_key)
.arg(entry.created_at.timestamp_millis())
.arg(response_id)
.ignore()
.cmd("EXPIRE")
.arg(&conversation_key)
.arg(ttl_seconds)
.ignore()
.cmd("ZADD")
.arg(&account_key)
.arg(entry.created_at.timestamp_millis())
.arg(response_id)
.ignore()
.cmd("EXPIRE")
.arg(&account_key)
.arg(ttl_seconds)
.ignore()
.cmd("ZREMRANGEBYSCORE")
.arg(conversation_key)
.arg("-inf")
.arg(cutoff_ms)
.ignore()
.cmd("ZREMRANGEBYSCORE")
.arg(&account_key)
.arg("-inf")
.arg(cutoff_ms)
.ignore();
let _: () = transaction.query_async(&mut connection).await?;
Ok(())
}
pub async fn get(
&self,
response_id: &str,
now: DateTime<Utc>,
ttl: Duration,
) -> RedisSessionAffinityStoreResult<Option<SessionAffinityEntry>> {
let mut connection = self.redis.manager();
let value: Option<String> = connection.get(self.response_key(response_id)).await?;
let Some(value) = value else {
return Ok(None);
};
let entry: SessionAffinityEntry = serde_json::from_str(&value)?;
if entry
.created_at
.checked_add_signed(ttl)
.is_none_or(|expires_at| expires_at <= now)
{
self.forget(response_id).await?;
return Ok(None);
}
Ok(Some(entry))
}
pub async fn latest_by_conversation(
&self,
conversation_id: &str,
max_age: Option<Duration>,
variant_hash: Option<&str>,
now: DateTime<Utc>,
ttl: Duration,
) -> RedisSessionAffinityStoreResult<Option<(String, SessionAffinityEntry)>> {
let conversation_key = self.conversation_key(conversation_id);
let mut connection = self.redis.manager();
let response_ids: Vec<String> = connection.zrevrange(&conversation_key, 0, -1).await?;
for response_id in response_ids {
let Some(entry) = self.get(&response_id, now, ttl).await? else {
let _: usize = connection.zrem(&conversation_key, &response_id).await?;
continue;
};
if entry.conversation_id != conversation_id {
let _: usize = connection.zrem(&conversation_key, &response_id).await?;
continue;
}
if variant_hash.is_some_and(|expected| entry.variant_hash.as_deref() != Some(expected))
{
continue;
}
if max_age.is_some_and(|max_age| now.signed_duration_since(entry.created_at) > max_age)
{
continue;
}
return Ok(Some((response_id, entry)));
}
Ok(None)
}
pub async fn replay_input(
&self,
response_id: &str,
head: &SessionAffinityEntry,
now: DateTime<Utc>,
ttl: Duration,
max_depth: u16,
max_total_bytes: u64,
) -> RedisSessionAffinityStoreResult<Option<Vec<Value>>> {
let Some(head_snapshot) = head.replay.as_ref() else {
return Ok(None);
};
if head_snapshot.depth == 0
|| head_snapshot.depth > max_depth
|| head_snapshot.total_bytes > max_total_bytes
{
return Ok(None);
}
let conversation_key = self.conversation_key(&head.conversation_id);
let mut connection = self.redis.manager();
let response_ids: Vec<String> = redis::cmd("ZREVRANGEBYSCORE")
.arg(&conversation_key)
.arg(head.created_at.timestamp_millis())
.arg("-inf")
.arg("LIMIT")
.arg(0)
.arg(max_depth)
.query_async(&mut connection)
.await?;
if !response_ids
.iter()
.any(|candidate| candidate == response_id)
{
return Ok(None);
}
let response_keys = response_ids
.iter()
.map(|candidate| self.response_key(candidate))
.collect::<Vec<_>>();
let values: Vec<Option<String>> = redis::cmd("MGET")
.arg(&response_keys)
.query_async(&mut connection)
.await?;
let mut entries = HashMap::with_capacity(values.len());
for (candidate, value) in response_ids.into_iter().zip(values) {
let Some(value) = value else {
continue;
};
let entry: SessionAffinityEntry = serde_json::from_str(&value)?;
if entry.conversation_id == head.conversation_id
&& entry
.created_at
.checked_add_signed(ttl)
.is_some_and(|expires_at| expires_at > now)
{
entries.insert(candidate, entry);
}
}
let mut current_id = response_id.to_string();
let mut expected_depth = head_snapshot.depth;
let mut expected_total_bytes = head_snapshot.total_bytes;
let mut nodes = Vec::with_capacity(usize::from(expected_depth));
loop {
let Some(entry) = entries.get(&current_id) else {
return Ok(None);
};
let Some(snapshot) = entry.replay.as_ref() else {
return Ok(None);
};
if snapshot.depth != expected_depth || snapshot.total_bytes != expected_total_bytes {
return Ok(None);
}
let node_bytes = replay_node_bytes(snapshot)?;
if node_bytes > expected_total_bytes {
return Ok(None);
}
nodes.push((snapshot.turn_input.clone(), snapshot.turn_output.clone()));
match snapshot.parent_response_id.as_deref() {
None if expected_depth == 1 && expected_total_bytes == node_bytes => break,
Some(parent_response_id) if expected_depth > 1 => {
expected_depth -= 1;
expected_total_bytes -= node_bytes;
current_id = parent_response_id.to_string();
}
None | Some(_) => return Ok(None),
}
}
nodes.reverse();
let mut full_input = Vec::new();
for (mut input, mut output) in nodes {
full_input.append(&mut input);
full_input.append(&mut output);
}
Ok(Some(full_input))
}
pub async fn forget(&self, response_id: &str) -> RedisSessionAffinityStoreResult<bool> {
let response_key = self.response_key(response_id);
let mut connection = self.redis.manager();
let value: Option<String> = connection.get(&response_key).await?;
let Some(value) = value else {
return Ok(false);
};
let entry: SessionAffinityEntry = serde_json::from_str(&value)?;
let mut transaction = redis::pipe();
transaction
.atomic()
.cmd("DEL")
.arg(response_key)
.ignore()
.cmd("ZREM")
.arg(self.conversation_key(&entry.conversation_id))
.arg(response_id)
.ignore()
.cmd("ZREM")
.arg(self.account_key(&entry.account_id))
.arg(response_id)
.ignore();
let _: () = transaction.query_async(&mut connection).await?;
Ok(true)
}
pub async fn forget_account(&self, account_id: &str) -> RedisSessionAffinityStoreResult<u64> {
let account_key = self.account_key(account_id);
let mut connection = self.redis.manager();
let response_ids: Vec<String> = connection.zrange(&account_key, 0, -1).await?;
if response_ids.is_empty() {
let _: usize = connection.del(account_key).await?;
return Ok(0);
}
let response_keys = response_ids
.iter()
.map(|response_id| self.response_key(response_id))
.collect::<Vec<_>>();
let values: Vec<Option<String>> = redis::cmd("MGET")
.arg(&response_keys)
.query_async(&mut connection)
.await?;
let mut conversation_members = Vec::with_capacity(response_ids.len());
for (response_id, value) in response_ids.iter().zip(values) {
let Some(value) = value else {
continue;
};
match serde_json::from_str::<SessionAffinityEntry>(&value) {
Ok(entry) if entry.account_id == account_id => {
conversation_members.push((response_id, entry.conversation_id));
}
Ok(_) => {
tracing::warn!(
response_id,
account_id,
"account affinity index points to another account"
);
}
Err(error) => {
tracing::warn!(
response_id,
account_id,
error = %error,
"invalid account affinity entry removed during account cleanup"
);
}
}
}
let mut transaction = redis::pipe();
transaction.atomic().cmd("DEL").arg(&response_keys).ignore();
for (response_id, conversation_id) in &conversation_members {
transaction
.cmd("ZREM")
.arg(self.conversation_key(conversation_id))
.arg(response_id)
.ignore();
}
transaction.cmd("DEL").arg(account_key).ignore();
let _: () = transaction.query_async(&mut connection).await?;
Ok(response_ids.len() as u64)
}
fn response_key(&self, response_id: &str) -> String {
self.redis.key(&format!("affinity:v2:resp:{response_id}"))
}
fn conversation_key(&self, conversation_id: &str) -> String {
self.redis
.key(&format!("affinity:v2:conv:{conversation_id}"))
}
fn account_key(&self, account_id: &str) -> String {
self.redis.key(&format!("affinity:v2:account:{account_id}"))
}
}
fn replay_node_bytes(
snapshot: &super::types::ResponseReplaySnapshot,
) -> Result<u64, serde_json::Error> {
serde_json::to_vec(&(&snapshot.turn_input, &snapshot.turn_output))
.map(|value| value.len() as u64)
}
+39
View File
@@ -0,0 +1,39 @@
//! 会话亲和领域类型。
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::upstream::openai::protocol::responses::PreviousResponseScope;
/// 会话亲和性条目。
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SessionAffinityEntry {
pub account_id: String,
pub conversation_id: String,
pub turn_state: Option<String>,
pub instructions_hash: Option<String>,
pub input_tokens: Option<u64>,
pub function_call_ids: Vec<String>,
pub variant_hash: Option<String>,
pub continuation_scope: PreviousResponseScope,
pub replay: Option<ResponseReplaySnapshot>,
pub created_at: DateTime<Utc>,
}
/// 单轮 completed response 的有界、无凭据重放增量。
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ResponseReplaySnapshot {
/// 上一轮由本代理记录的 completed response。
pub parent_response_id: Option<String>,
/// 本轮客户端新增的输入,已剥离账号绑定字段。
pub turn_input: Vec<Value>,
/// 本轮上游输出,已剥离账号绑定字段和上游对象 ID。
pub turn_output: Vec<Value>,
/// 从会话起点到本节点的增量节点数。
pub depth: u16,
/// 从会话起点到本节点的 JSON 编码累计字节数。
pub total_bytes: u64,
}
+82
View File
@@ -0,0 +1,82 @@
//! 请求级账号候选快照与尝试账本。
use std::collections::{BTreeSet, VecDeque};
use chrono::Utc;
use crate::fleet::pool::{
AccountAcquireRequest, AccountCandidateLease, AccountLease, AccountPoolService,
};
/// 请求开始时冻结的完整候选集,以及候选的最终处理状态。
pub(in crate::dispatch) struct AccountAttemptLedger {
model: String,
candidate_ids: Vec<String>,
pending: VecDeque<String>,
busy: VecDeque<String>,
attempted: BTreeSet<String>,
state_excluded: BTreeSet<String>,
}
impl AccountAttemptLedger {
pub(in crate::dispatch) async fn freeze(
account_pool: &AccountPoolService,
request: &AccountAcquireRequest,
) -> Self {
let candidate_ids = account_pool.candidate_snapshot(request).await;
Self {
model: request.model.clone(),
pending: candidate_ids.iter().cloned().collect(),
candidate_ids,
busy: VecDeque::new(),
attempted: BTreeSet::new(),
state_excluded: BTreeSet::new(),
}
}
pub(in crate::dispatch) async fn acquire_next(
&mut self,
account_pool: &AccountPoolService,
) -> Option<AccountLease> {
loop {
while let Some(account_id) = self.pending.pop_front() {
match account_pool
.acquire_candidate(&self.model, &account_id, Utc::now())
.await
{
AccountCandidateLease::Acquired(lease) => {
self.attempted.insert(account_id);
return Some(*lease);
}
AccountCandidateLease::Busy => self.busy.push_back(account_id),
AccountCandidateLease::Unavailable => {
self.state_excluded.insert(account_id);
}
}
}
if self.busy.is_empty() {
debug_assert_eq!(
self.candidate_ids.len(),
self.attempted.len() + self.state_excluded.len()
);
return None;
}
account_pool.wait_for_candidate_change().await;
std::mem::swap(&mut self.pending, &mut self.busy);
}
}
pub(in crate::dispatch) fn candidate_count(&self) -> usize {
self.candidate_ids.len()
}
pub(in crate::dispatch) fn attempted_count(&self) -> usize {
self.attempted.len()
}
pub(in crate::dispatch) fn state_excluded_count(&self) -> usize {
self.state_excluded.len()
}
}
+718
View File
@@ -0,0 +1,718 @@
//! Shared upstream error classification for dispatch routes.
use chrono::{DateTime, Duration, Utc};
use serde_json::{json, Map, Value};
use thiserror::Error;
use crate::{
dispatch::{
recovery::exhaustion::{ExhaustedAccount, ExhaustedAccountKind, ExhaustedAccountRef},
stream::sse_failure::{sse_failure_error_body, stream_failure_http_status},
},
upstream::openai::{
protocol::{responses::ResponsesSseFailure, sse::SseError},
transport::{CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics},
},
};
#[derive(Clone, Debug)]
pub(crate) struct DispatchErrorMetadata {
pub failure_class: DispatchFailureClass,
pub exhausted_count: Option<usize>,
pub upstream_error: Option<String>,
pub upstream_status: Option<u16>,
pub diagnostics: Option<CodexUpstreamDiagnostics>,
}
#[derive(Clone, Copy, Debug)]
pub(crate) enum DispatchFailureClass {
NoAvailableAccounts,
QuotaExhausted,
RateLimited,
Expired,
Disabled,
Banned,
CloudflareChallenge,
CloudflarePathBlocked,
ModelUnsupported,
UpstreamUnavailable,
ContinuationBusy,
HistoryUnavailable,
Upstream,
InvalidSse,
MissingCompleted,
EmptyUpstreamResponse,
ResponseFailed,
}
impl DispatchFailureClass {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::NoAvailableAccounts => "no_available_accounts",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Expired => "expired",
Self::Disabled => "disabled",
Self::Banned => "banned",
Self::CloudflareChallenge => "cloudflare_challenge",
Self::CloudflarePathBlocked => "cloudflare_path_blocked",
Self::ModelUnsupported => "model_unsupported",
Self::UpstreamUnavailable => "upstream_unavailable",
Self::ContinuationBusy => "continuation_busy",
Self::HistoryUnavailable => "history_unavailable",
Self::Upstream => "upstream",
Self::InvalidSse => "invalid_sse",
Self::MissingCompleted => "missing_completed",
Self::EmptyUpstreamResponse => "empty_upstream_response",
Self::ResponseFailed => "response_failed",
}
}
}
impl DispatchErrorMetadata {
pub(crate) fn no_available_accounts() -> Self {
Self {
failure_class: DispatchFailureClass::NoAvailableAccounts,
exhausted_count: None,
upstream_error: None,
upstream_status: None,
diagnostics: None,
}
}
pub(crate) fn exhausted_ref(exhausted: ExhaustedAccountRef<'_>) -> Self {
Self {
failure_class: DispatchFailureClass::from(exhausted.kind),
exhausted_count: Some(exhausted.count),
upstream_error: Some(exhausted.upstream_error.to_string()),
upstream_status: None,
diagnostics: None,
}
}
pub(crate) fn upstream(error: &CodexClientError) -> Self {
Self {
failure_class: DispatchFailureClass::Upstream,
exhausted_count: None,
upstream_error: Some(upstream_error_body(error)),
upstream_status: match error {
CodexClientError::Upstream { status, .. } => Some(status.as_u16()),
_ => None,
},
diagnostics: upstream_error_diagnostics(error).cloned(),
}
}
pub(crate) fn simple(failure_class: DispatchFailureClass) -> Self {
Self {
failure_class,
exhausted_count: None,
upstream_error: None,
upstream_status: None,
diagnostics: None,
}
}
}
impl From<ExhaustedAccountKind> for DispatchFailureClass {
fn from(kind: ExhaustedAccountKind) -> Self {
match kind {
ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted,
ExhaustedAccountKind::RateLimited => Self::RateLimited,
ExhaustedAccountKind::Expired => Self::Expired,
ExhaustedAccountKind::Disabled => Self::Disabled,
ExhaustedAccountKind::Banned => Self::Banned,
ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge,
ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked,
ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported,
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable,
}
}
}
pub(crate) fn insert_dispatch_error_metadata(
object: &mut Map<String, Value>,
metadata: DispatchErrorMetadata,
) {
object.insert(
"failureClass".to_string(),
Value::String(metadata.failure_class.as_str().to_string()),
);
if let Some(count) = metadata.exhausted_count {
object.insert("exhaustedCount".to_string(), json!(count));
}
if let Some(error) = metadata.upstream_error {
object.insert("upstreamError".to_string(), Value::String(error));
}
if let Some(status) = metadata.upstream_status {
object.insert("upstreamStatus".to_string(), json!(status));
}
if let Some(diagnostics) = metadata.diagnostics {
insert_upstream_diagnostics_metadata(object, &diagnostics);
}
}
pub(crate) fn insert_upstream_diagnostics_metadata(
object: &mut Map<String, Value>,
diagnostics: &CodexUpstreamDiagnostics,
) {
if diagnostics.is_empty() {
return;
}
if let Some(status_code) = diagnostics.status_code {
object.insert("upstreamStatus".to_string(), json!(status_code));
}
if let Some(request_id) = &diagnostics.request_id {
object.insert(
"upstreamRequestId".to_string(),
Value::String(request_id.clone()),
);
}
if !diagnostics.trace_headers.is_empty() {
object.insert(
"upstreamTraceHeaders".to_string(),
json!(diagnostics.trace_headers),
);
}
if let Some(cf_ray) = diagnostics.cf_ray() {
object.insert("cfRay".to_string(), Value::String(cf_ray.to_string()));
}
}
pub(crate) fn upstream_error_diagnostics(
error: &CodexClientError,
) -> Option<&CodexUpstreamDiagnostics> {
match error {
CodexClientError::Upstream { diagnostics, .. } => Some(diagnostics),
_ => None,
}
}
pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::Upstream { status, .. } if status_code_is_rate_limited(status.as_u16())
)
}
pub(crate) fn is_retryable_upstream_5xx_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::Upstream { status, .. }
if status_code_is_transient_upstream(status.as_u16())
)
}
pub(crate) fn is_retryable_account_transport_error(error: &CodexClientError) -> bool {
match error {
CodexClientError::Http(error) => error.is_connect() || error.is_timeout(),
CodexClientError::WebSocket(
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::Transport(_)
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ClosedBeforeTerminal
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. }
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::InitialEventTimeout { .. },
) => true,
_ => is_retryable_upstream_5xx_error(error),
}
}
pub(crate) fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::Upstream { status, .. } if status_code_is_quota_exhausted(status.as_u16())
)
}
pub(crate) fn is_model_unsupported_upstream_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::Upstream { status, body, .. }
if status.is_client_error()
&& !matches!(status.as_u16(), 401 | 402 | 403 | 404 | 429)
&& is_model_unsupported_signal(body)
)
}
pub(crate) fn is_history_recovery_upstream_error(error: &CodexClientError) -> bool {
upstream_error_code(error).is_some_and(|code| is_history_recovery_code(&code))
|| matches!(
error,
CodexClientError::WebSocket(
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable { reason }
) if !matches!(
reason,
crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy
)
)
}
pub(crate) fn is_continuation_busy_error(error: &CodexClientError) -> bool {
matches!(
error,
CodexClientError::WebSocket(
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable {
reason: crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy,
}
)
)
}
pub(crate) fn upstream_error_body(error: &CodexClientError) -> String {
match error {
CodexClientError::Upstream { body, .. } => body.clone(),
error => error.to_string(),
}
}
pub(crate) fn upstream_error_set_cookie_headers(error: &CodexClientError) -> &[String] {
match error {
CodexClientError::Upstream {
set_cookie_headers, ..
} => set_cookie_headers,
_ => &[],
}
}
pub(crate) fn is_model_unsupported_signal(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("model_not_supported")
|| value.contains("model_not_available")
|| (value.contains("model")
&& (value.contains("not supported")
|| value.contains("not available")
|| value.contains("not_supported")
|| value.contains("not_available")))
}
pub(crate) fn is_history_recovery_code(code: &str) -> bool {
matches!(
code,
"previous_response_not_found"
| "invalid_encrypted_content"
| "missing_tool_output"
| "no_tool_output"
)
}
pub(crate) fn upstream_error_code(error: &CodexClientError) -> Option<String> {
let CodexClientError::Upstream { body, .. } = error else {
return None;
};
let value = serde_json::from_str::<Value>(body).ok()?;
value
.pointer("/response/error/code")
.or_else(|| value.pointer("/error/code"))
.or_else(|| value.get("code"))
.and_then(Value::as_str)
.map(ToString::to_string)
}
pub(crate) fn rate_limit_cooldown_until(
error: &CodexClientError,
now: DateTime<Utc>,
) -> DateTime<Utc> {
let retry_after_seconds = match error {
CodexClientError::Upstream {
retry_after_seconds,
..
} => retry_after_seconds.unwrap_or(60),
_ => 60,
};
now + Duration::seconds(retry_after_seconds.min(i64::MAX as u64) as i64)
}
pub(crate) fn upstream_error_http_status(error: &CodexClientError) -> u16 {
match error {
CodexClientError::Upstream { status, .. } => status.as_u16(),
_ => 502,
}
}
pub(crate) fn backend_transport_name(transport: CodexBackendTransport) -> &'static str {
match transport {
CodexBackendTransport::HttpSse => "http_sse",
CodexBackendTransport::WebSocket => "websocket",
}
}
fn status_code_is_rate_limited(status_code: u16) -> bool {
status_code == 429
}
fn status_code_is_quota_exhausted(status_code: u16) -> bool {
status_code == 402
}
fn status_code_is_transient_upstream(status_code: u16) -> bool {
matches!(status_code, 500..=599)
}
/// Responses 调度错误。
#[derive(Debug, Error)]
pub enum ResponseDispatchError {
#[error("no active account is available")]
NoActiveAccount,
#[error("all accounts exhausted by quota")]
QuotaExhausted {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by rate limit")]
RateLimited {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by expired auth")]
Expired {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by disabled auth")]
Disabled {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by banned auth")]
Banned {
count: usize,
upstream_error: String,
status_code: u16,
},
#[error("all accounts exhausted by Cloudflare challenge")]
CloudflareChallenge {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by Cloudflare path-block")]
CloudflarePathBlocked {
count: usize,
upstream_error: String,
},
#[error("all accounts exhausted by unsupported model")]
ModelUnsupported {
count: usize,
upstream_error: String,
},
#[error("all account candidates failed with transient upstream errors")]
UpstreamUnavailable {
count: usize,
upstream_error: String,
},
#[error("the previous response connection is busy; retry the request")]
ContinuationBusy,
#[error("previous response context is unavailable: {upstream_error}")]
HistoryUnavailable { upstream_error: String },
#[error("upstream request failed: {0}")]
Upstream(#[from] CodexClientError),
#[error("invalid upstream SSE response: {0}")]
InvalidSse(#[from] SseError),
#[error("upstream response did not include response.completed")]
MissingCompleted,
#[error("upstream response did not include visible output")]
EmptyUpstreamResponse,
#[error("upstream response failed: {0:?}")]
Failed(ResponsesSseFailure),
}
impl ResponseDispatchError {
pub(crate) fn from_exhausted_account(exhausted: ExhaustedAccount) -> Self {
match exhausted.kind {
ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::RateLimited => Self::RateLimited {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::Expired => Self::Expired {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::Disabled => Self::Disabled {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::Banned => Self::Banned {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
status_code: exhausted.status_code.unwrap_or(403),
},
ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable {
count: exhausted.count,
upstream_error: exhausted.upstream_error,
},
}
}
pub fn http_status_code(&self) -> u16 {
match self {
Self::NoActiveAccount => 503,
Self::QuotaExhausted { .. } => 429,
Self::RateLimited { .. } => 429,
Self::Expired { .. } | Self::Disabled { .. } => 401,
Self::Banned { status_code, .. } => *status_code,
Self::CloudflareChallenge { .. }
| Self::CloudflarePathBlocked { .. }
| Self::InvalidSse(_)
| Self::MissingCompleted
| Self::EmptyUpstreamResponse => 502,
Self::UpstreamUnavailable { .. } => 503,
Self::ContinuationBusy => 503,
Self::Failed(failure) => stream_failure_http_status(failure),
Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400,
Self::Upstream(error) => upstream_error_http_status(error),
}
}
pub fn client_http_status_code(&self) -> u16 {
match self {
Self::Upstream(_) => client_upstream_http_status_code(self.http_status_code()),
_ => self.http_status_code(),
}
}
pub(crate) fn metadata(&self) -> DispatchErrorMetadata {
match self {
Self::NoActiveAccount => DispatchErrorMetadata::no_available_accounts(),
Self::QuotaExhausted {
count,
upstream_error,
} => Self::exhausted_metadata(
ExhaustedAccountKind::QuotaExhausted,
*count,
upstream_error,
),
Self::RateLimited {
count,
upstream_error,
} => {
Self::exhausted_metadata(ExhaustedAccountKind::RateLimited, *count, upstream_error)
}
Self::Expired {
count,
upstream_error,
} => Self::exhausted_metadata(ExhaustedAccountKind::Expired, *count, upstream_error),
Self::Disabled {
count,
upstream_error,
} => Self::exhausted_metadata(ExhaustedAccountKind::Disabled, *count, upstream_error),
Self::Banned {
count,
upstream_error,
..
} => Self::exhausted_metadata(ExhaustedAccountKind::Banned, *count, upstream_error),
Self::CloudflareChallenge {
count,
upstream_error,
} => Self::exhausted_metadata(
ExhaustedAccountKind::CloudflareChallenge,
*count,
upstream_error,
),
Self::CloudflarePathBlocked {
count,
upstream_error,
} => Self::exhausted_metadata(
ExhaustedAccountKind::CloudflarePathBlocked,
*count,
upstream_error,
),
Self::ModelUnsupported {
count,
upstream_error,
} => Self::exhausted_metadata(
ExhaustedAccountKind::ModelUnsupported,
*count,
upstream_error,
),
Self::UpstreamUnavailable {
count,
upstream_error,
} => Self::exhausted_metadata(
ExhaustedAccountKind::UpstreamUnavailable,
*count,
upstream_error,
),
Self::ContinuationBusy => {
DispatchErrorMetadata::simple(DispatchFailureClass::ContinuationBusy)
}
Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata {
failure_class: DispatchFailureClass::HistoryUnavailable,
exhausted_count: None,
upstream_error: Some(upstream_error.clone()),
upstream_status: Some(400),
diagnostics: None,
},
Self::Upstream(error) => DispatchErrorMetadata::upstream(error),
Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse),
Self::MissingCompleted => {
DispatchErrorMetadata::simple(DispatchFailureClass::MissingCompleted)
}
Self::EmptyUpstreamResponse => {
DispatchErrorMetadata::simple(DispatchFailureClass::EmptyUpstreamResponse)
}
Self::Failed(failure) => DispatchErrorMetadata {
failure_class: DispatchFailureClass::ResponseFailed,
exhausted_count: None,
upstream_error: Some(sse_failure_error_body(failure)),
upstream_status: None,
diagnostics: None,
},
}
}
pub(crate) fn exhausted_account(&self) -> Option<ExhaustedAccountRef<'_>> {
match self {
Self::QuotaExhausted {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::QuotaExhausted,
count: *count,
upstream_error,
}),
Self::RateLimited {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::RateLimited,
count: *count,
upstream_error,
}),
Self::Expired {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::Expired,
count: *count,
upstream_error,
}),
Self::Disabled {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::Disabled,
count: *count,
upstream_error,
}),
Self::Banned {
count,
upstream_error,
..
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::Banned,
count: *count,
upstream_error,
}),
Self::CloudflareChallenge {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::CloudflareChallenge,
count: *count,
upstream_error,
}),
Self::CloudflarePathBlocked {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::CloudflarePathBlocked,
count: *count,
upstream_error,
}),
Self::ModelUnsupported {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::ModelUnsupported,
count: *count,
upstream_error,
}),
Self::UpstreamUnavailable {
count,
upstream_error,
} => Some(ExhaustedAccountRef {
kind: ExhaustedAccountKind::UpstreamUnavailable,
count: *count,
upstream_error,
}),
_ => None,
}
}
fn exhausted_metadata(
kind: ExhaustedAccountKind,
count: usize,
upstream_error: &str,
) -> DispatchErrorMetadata {
DispatchErrorMetadata::exhausted_ref(ExhaustedAccountRef {
kind,
count,
upstream_error,
})
}
}
pub(crate) fn client_upstream_http_status_code(status: u16) -> u16 {
match status {
400..=499 | 503 => status,
_ => 502,
}
}
/// Responses live SSE body stream error.
#[derive(Debug, Error)]
pub enum ResponseDispatchStreamError {
#[error("upstream stream failed: {0}")]
Upstream(#[from] CodexClientError),
}
pub(super) fn dispatch_error_metadata(
error: impl std::fmt::Display,
stream: bool,
compact: bool,
transport: Option<&str>,
) -> Value {
let mut metadata = serde_json::json!({
"stream": stream,
"failed": true,
"errorKind": "dispatch",
"error": error.to_string(),
});
let Some(object) = metadata.as_object_mut() else {
return metadata;
};
if compact {
object.insert("compact".to_string(), Value::Bool(true));
}
if let Some(transport) = transport {
object.insert(
"transport".to_string(),
Value::String(transport.to_string()),
);
}
metadata
}
pub(super) fn enrich_response_dispatch_error_metadata(
metadata: &mut Value,
error: &ResponseDispatchError,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_dispatch_error_metadata(object, error.metadata());
}
+10
View File
@@ -0,0 +1,10 @@
//! 请求调度模块(账号选择、回退、恢复、会话亲和性)。
pub mod affinity;
mod attempts;
pub mod errors;
mod recording;
pub mod recovery;
pub(crate) mod service;
pub(crate) mod stream;
pub mod upstream_call;
+675
View File
@@ -0,0 +1,675 @@
use std::time::Instant;
use serde_json::{json, Map, Value};
use crate::{
dispatch::errors::{
backend_transport_name, insert_dispatch_error_metadata,
insert_upstream_diagnostics_metadata, upstream_error_http_status, DispatchErrorMetadata,
},
infra::time::elapsed_millis_i64,
telemetry::{
ops::types::OpsErrorLog,
recorder::{
enrich_event_route_metadata, enrich_usage_record_identity,
event_kind as response_event_kind, reasoning_effort_from_request,
record_dispatch_error_event, DispatchErrorLogRecord, Recorder,
},
usage::types::UsageRecordLevel,
},
upstream::openai::{
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
transport::{
CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
WebSocketPoolDecision,
},
},
};
use super::{
errors::{
dispatch_error_metadata, enrich_response_dispatch_error_metadata, ResponseDispatchError,
},
stream::{
live::{latest_response_id, LiveResponseStreamContext},
sse_failure::{status_code_for_stream_failure, stream_failure_metadata},
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
},
};
pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
pub(super) recorder: &'a Recorder,
pub(super) request_id: &'a str,
pub(super) account_id: &'a str,
pub(super) account_email: Option<&'a str>,
pub(super) route: &'a str,
pub(super) model: &'a str,
pub(super) started_at: Instant,
pub(super) stream: bool,
pub(super) transport: CodexBackendTransport,
pub(super) request: &'a CodexResponsesRequest,
pub(super) error: &'a CodexClientError,
pub(super) trace: &'a ResponseDispatchTrace,
pub(super) attempt: Option<&'a ResponseDispatchAttempt>,
}
pub(super) struct ResponseStreamFailureEventRecord<'a> {
pub(super) recorder: &'a Recorder,
pub(super) request_id: &'a str,
pub(super) account_id: &'a str,
pub(super) route: &'a str,
pub(super) model: &'a str,
pub(super) requested_model: &'a str,
pub(super) started_at: Instant,
pub(super) transport: CodexBackendTransport,
pub(super) request: &'a CodexResponsesRequest,
pub(super) failure: &'a ResponsesSseFailure,
pub(super) diagnostics: &'a CodexUpstreamDiagnostics,
pub(super) rate_limit_headers: &'a [(String, String)],
pub(super) prefetched: &'a [u8],
pub(super) trace: &'a ResponseDispatchTrace,
pub(super) attempt: &'a ResponseDispatchAttempt,
}
pub(super) struct ResponseDispatchErrorEventRecord<'a> {
pub(super) recorder: &'a Recorder,
pub(super) request_id: &'a str,
pub(super) client_api_key_id: Option<&'a str>,
pub(super) account_id: Option<&'a str>,
pub(super) route: &'a str,
pub(super) model: &'a str,
pub(super) started_at: Instant,
pub(super) stream: bool,
pub(super) compact: bool,
pub(super) transport: Option<&'a str>,
pub(super) error: &'a ResponseDispatchError,
}
pub(super) struct ResponseDispatchErrorDetails<'a> {
pub(super) client_api_key_id: Option<&'a str>,
pub(super) account_id: Option<&'a str>,
pub(super) stream: bool,
pub(super) compact: bool,
pub(super) transport: Option<&'a str>,
}
pub(super) async fn record_response_dispatch_error_event(
record: ResponseDispatchErrorEventRecord<'_>,
) {
let mut metadata = dispatch_error_metadata(
record.error,
record.stream,
record.compact,
record.transport,
);
enrich_response_dispatch_error_metadata(&mut metadata, record.error);
if let Some(object) = metadata.as_object_mut() {
insert_response_status_metadata_object(
object,
i64::from(record.error.http_status_code()),
dispatch_error_client_status_code(record.error),
dispatch_error_upstream_status_code(record.error),
);
}
record_dispatch_error_event(DispatchErrorLogRecord {
recorder: record.recorder,
request_id: record.request_id,
client_api_key_id: record.client_api_key_id,
provider: Some("openai"),
account_id: record.account_id,
route: record.route,
model: record.model,
started_at: record.started_at,
status_code: i64::from(record.error.http_status_code()),
message: "v1 responses dispatch failed",
metadata,
})
.await;
}
pub(super) async fn record_response_upstream_error_event(
record: ResponseUpstreamErrorEventRecord<'_>,
) {
let event_status_code = i64::from(upstream_error_http_status(record.error));
let mut metadata = dispatch_error_metadata(
record.error,
record.stream,
false,
Some(backend_transport_name(record.transport)),
);
enrich_event_route_metadata(&mut metadata, record.route);
let mut event = OpsErrorLog::new("v1.response", "v1 responses upstream request failed");
event.request_id = Some(record.request_id.to_string());
event.client_api_key_id = record.request.client_api_key_id.clone();
event.provider = Some("openai".to_string());
event.account_id = Some(record.account_id.to_string());
event.route = Some(record.route.to_string());
event.model = Some(record.model.to_string());
event.status_code = Some(event_status_code);
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
if let Some(object) = metadata.as_object_mut() {
insert_dispatch_error_metadata(object, DispatchErrorMetadata::upstream(record.error));
insert_response_status_metadata_object(
object,
event_status_code,
upstream_failure_client_status_code(event_status_code),
Some(event_status_code),
);
insert_response_request_summary_object(object, record.request, record.transport);
insert_response_trace_metadata_object(object, record.trace.attempts(), record.attempt);
if let Some(account_email) = record
.account_email
.map(str::trim)
.filter(|value| !value.is_empty())
{
object.insert(
"accountEmail".to_string(),
Value::String(account_email.to_string()),
);
}
}
enrich_usage_record_identity(
&mut metadata,
Some(record.request.model()),
record.model,
record.request.client_ip.as_deref(),
record.request.client_user_agent.as_deref(),
reasoning_effort_from_request(record.request),
record.request.service_tier(),
);
event.metadata = metadata;
if let Err(error) = record.recorder.record_error(event).await {
tracing::error!(account_id = %record.account_id, error = %error, "failed to record upstream error event");
}
}
pub(super) async fn record_prefetched_response_stream_failure_event(
record: ResponseStreamFailureEventRecord<'_>,
) {
let event_status_code = status_code_for_stream_failure(record.failure);
let mut metadata = stream_failure_metadata(record.failure, None);
if let Some(object) = metadata.as_object_mut() {
if record.transport == CodexBackendTransport::WebSocket {
object.insert(
"transport".to_string(),
Value::String("websocket".to_string()),
);
}
insert_response_status_metadata_object(
object,
event_status_code,
200,
diagnostics_status_code(record.diagnostics),
);
insert_response_request_summary_object(object, record.request, record.transport);
object.insert("requestBody".to_string(), json!(record.request));
object.insert(
"responseBody".to_string(),
Value::String(String::from_utf8_lossy(record.prefetched).to_string()),
);
insert_upstream_diagnostics_metadata(object, record.diagnostics);
insert_response_trace_metadata_object(
object,
record.trace.attempts(),
Some(record.attempt),
);
}
enrich_event_route_metadata(&mut metadata, record.route);
enrich_usage_record_identity(
&mut metadata,
Some(record.requested_model),
record.model,
record.request.client_ip.as_deref(),
record.request.client_user_agent.as_deref(),
reasoning_effort_from_request(record.request),
record.request.service_tier(),
);
let mut event = OpsErrorLog::new(
response_event_kind(record.route),
"v1 responses stream failed",
);
event.request_id = Some(record.request_id.to_string());
event.client_api_key_id = record.request.client_api_key_id.clone();
event.provider = Some("openai".to_string());
event.account_id = Some(record.account_id.to_string());
event.route = Some(record.route.to_string());
event.model = Some(record.model.to_string());
event.status_code = Some(event_status_code);
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
if !record.rate_limit_headers.is_empty() {
if let Some(object) = metadata.as_object_mut() {
object.insert(
"rateLimitHeaders".to_string(),
serde_json::json!(record.rate_limit_headers),
);
}
}
event.metadata = metadata;
if let Err(error) = record.recorder.record_error(event).await {
tracing::error!(account_id = %record.account_id, error = %error, "failed to record prefetched stream error event");
}
}
fn ensure_stream_metadata_flag(metadata: &mut Value) {
let Some(object) = metadata.as_object_mut() else {
*metadata = serde_json::json!({ "stream": true });
return;
};
object
.entry("stream".to_string())
.or_insert(Value::Bool(true));
}
fn enrich_live_response_stream_metadata(
context: &LiveResponseStreamContext,
rate_limit_headers: &[(String, String)],
metadata: &mut Value,
status_code: i64,
body: &str,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
object
.entry("transport".to_string())
.or_insert_with(|| Value::String(backend_transport_name(context.transport).to_string()));
if !rate_limit_headers.is_empty() {
object
.entry("rateLimitHeaders".to_string())
.or_insert_with(|| serde_json::json!(rate_limit_headers));
}
if let Some(decision) = context.websocket_pool_decision {
object
.entry("websocketPool".to_string())
.or_insert_with(|| decision.metadata_value());
}
insert_response_status_metadata_object(
object,
status_code,
200,
diagnostics_status_code(&context.diagnostics),
);
insert_response_request_summary_object(object, &context.request, context.transport);
insert_upstream_diagnostics_metadata(object, &context.diagnostics);
insert_response_trace_metadata_object(object, &context.attempts, Some(&context.attempt));
object
.entry("requestBody".to_string())
.or_insert_with(|| serde_json::json!(context.request));
object
.entry("responseBody".to_string())
.or_insert_with(|| Value::String(body.to_string()));
}
pub(super) fn insert_response_upstream_diagnostics(
metadata: &mut Value,
diagnostics: &CodexUpstreamDiagnostics,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_upstream_diagnostics_metadata(object, diagnostics);
}
pub(super) fn insert_response_status_metadata(
metadata: &mut Value,
event_status_code: i64,
client_status_code: i64,
upstream_status_code: Option<i64>,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_response_status_metadata_object(
object,
event_status_code,
client_status_code,
upstream_status_code,
);
}
pub(super) fn insert_response_trace_metadata(
metadata: &mut Value,
trace: &ResponseDispatchTrace,
current_attempt: Option<&ResponseDispatchAttempt>,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_response_trace_metadata_object(object, trace.attempts(), current_attempt);
}
fn insert_response_attempt_metadata_object(
object: &mut serde_json::Map<String, Value>,
attempt: Option<&ResponseDispatchAttempt>,
) {
let Some(attempt) = attempt else {
return;
};
object.insert("attemptIndex".to_string(), json!(attempt.index()));
object.insert(
"attemptAccountId".to_string(),
Value::String(attempt.account_id().to_string()),
);
}
fn insert_response_trace_metadata_object(
object: &mut Map<String, Value>,
attempts: &[ResponseDispatchAttempt],
current_attempt: Option<&ResponseDispatchAttempt>,
) {
insert_response_attempt_metadata_object(object, current_attempt);
object.insert("attemptCount".to_string(), json!(attempts.len()));
object.insert(
"attempts".to_string(),
Value::Array(
attempts
.iter()
.map(|attempt| {
json!({
"index": attempt.index(),
"accountId": attempt.account_id(),
})
})
.collect(),
),
);
}
fn insert_response_status_metadata_object(
object: &mut Map<String, Value>,
event_status_code: i64,
client_status_code: i64,
upstream_status_code: Option<i64>,
) {
object.insert("eventStatusCode".to_string(), json!(event_status_code));
object.insert("clientStatusCode".to_string(), json!(client_status_code));
if let Some(upstream_status_code) = upstream_status_code {
object.insert(
"upstreamStatusCode".to_string(),
json!(upstream_status_code),
);
}
}
fn insert_response_request_summary_object(
object: &mut Map<String, Value>,
request: &CodexResponsesRequest,
transport: CodexBackendTransport,
) {
object.insert(
"requestSummary".to_string(),
response_request_summary(request, transport),
);
}
fn response_request_summary(
request: &CodexResponsesRequest,
transport: CodexBackendTransport,
) -> Value {
let body = request.body();
let input = body.get("input");
let tools = body.get("tools");
json!({
"model": request.model(),
"stream": request.stream(),
"store": request.store(),
"transport": backend_transport_name(transport),
"inputType": json_value_kind(input),
"inputItemsCount": input.and_then(Value::as_array).map(Vec::len),
"toolsType": json_value_kind(tools),
"toolsCount": tools.and_then(Value::as_array).map(Vec::len),
"topLevelFields": body.keys().cloned().collect::<Vec<_>>(),
"previousResponseIdPresent": request.previous_response_id().is_some(),
"serviceTier": request.service_tier(),
"localTransport": {
"useWebsocket": request.use_websocket,
"forceHttpSse": request.force_http_sse,
},
})
}
fn json_value_kind(value: Option<&Value>) -> &'static str {
match value {
None => "missing",
Some(Value::Null) => "null",
Some(Value::Bool(_)) => "boolean",
Some(Value::Number(_)) => "number",
Some(Value::String(_)) => "string",
Some(Value::Array(_)) => "array",
Some(Value::Object(_)) => "object",
}
}
fn diagnostics_status_code(diagnostics: &CodexUpstreamDiagnostics) -> Option<i64> {
diagnostics.status_code.map(i64::from)
}
fn dispatch_error_upstream_status_code(error: &ResponseDispatchError) -> Option<i64> {
match error {
ResponseDispatchError::Upstream(error) => {
Some(i64::from(upstream_error_http_status(error)))
}
_ => None,
}
}
fn dispatch_error_client_status_code(error: &ResponseDispatchError) -> i64 {
i64::from(error.client_http_status_code())
}
fn upstream_failure_client_status_code(upstream_status: i64) -> i64 {
i64::from(crate::dispatch::errors::client_upstream_http_status_code(
u16::try_from(upstream_status).unwrap_or(502),
))
}
pub(super) async fn record_live_response_stream_event(
context: &LiveResponseStreamContext,
status_code: i64,
level: UsageRecordLevel,
message: &str,
mut metadata: Value,
rate_limit_headers: &[(String, String)],
body: &str,
) {
let effective_model = context
.response_metadata
.effective_model
.as_deref()
.unwrap_or(&context.display_model);
ensure_stream_metadata_flag(&mut metadata);
if let Some(object) = metadata.as_object_mut() {
object.insert(
"effectiveModel".to_string(),
Value::String(effective_model.to_string()),
);
object.insert(
"modelsEtag".to_string(),
context
.response_metadata
.models_etag
.as_ref()
.map_or(Value::Null, |etag| Value::String(etag.clone())),
);
object.insert(
"reasoningIncluded".to_string(),
Value::Bool(context.response_metadata.reasoning_included),
);
}
enrich_event_route_metadata(&mut metadata, &context.route);
enrich_live_response_stream_metadata(
context,
rate_limit_headers,
&mut metadata,
status_code,
body,
);
log_live_response_stream_finalized(context, status_code, level, message, &metadata, body);
enrich_usage_record_identity(
&mut metadata,
Some(&context.requested_model),
effective_model,
context.client_ip.as_deref(),
context.request.client_user_agent.as_deref(),
reasoning_effort_from_request(&context.request),
context.request.service_tier(),
);
if is_success_usage_event(level, status_code) {
crate::telemetry::recorder::record_response_event(
crate::telemetry::usage::types::ResponseUsageRecord {
recorder: &context.recorder,
request_id: &context.request_id,
client_api_key_id: context.request.client_api_key_id.as_deref(),
account_id: &context.account_id,
route: &context.route,
model: effective_model,
requested_model: Some(&context.requested_model),
client_ip: context.client_ip.as_deref(),
client_user_agent: context.request.client_user_agent.as_deref(),
reasoning_effort: reasoning_effort_from_request(&context.request),
service_tier: context.request.service_tier(),
started_at: context.started_at,
status_code,
message,
metadata,
rate_limit_headers,
},
)
.await;
} else {
let mut event = OpsErrorLog::new(response_event_kind(&context.route), message);
event.request_id = Some(context.request_id.clone());
event.client_api_key_id = context.request.client_api_key_id.clone();
event.provider = Some("openai".to_string());
event.account_id = Some(context.account_id.clone());
event.route = Some(context.route.clone());
event.model = Some(effective_model.to_string());
event.status_code = Some(status_code);
event.latency_ms = Some(elapsed_millis_i64(context.started_at));
event.metadata = metadata;
if let Err(error) = context.recorder.record_error(event).await {
tracing::error!(account_id = %context.account_id, error = %error, "failed to record live response stream error event");
}
}
}
fn is_success_usage_event(level: UsageRecordLevel, status_code: i64) -> bool {
level != UsageRecordLevel::Error && status_code < 400
}
fn log_live_response_stream_finalized(
context: &LiveResponseStreamContext,
status_code: i64,
level: UsageRecordLevel,
message: &str,
metadata: &Value,
body: &str,
) {
let response_id = metadata_string_field(metadata, "responseId")
.map(ToString::to_string)
.or_else(|| latest_response_id(body));
let first_token_ms = metadata.get("firstTokenMs").and_then(Value::as_i64);
let websocket_pool_kind = context
.websocket_pool_decision
.map(WebSocketPoolDecision::kind);
let websocket_pool_reason = context
.websocket_pool_decision
.and_then(WebSocketPoolDecision::reason);
let completed = metadata.get("completed").and_then(Value::as_bool);
let failed = metadata.get("failed").and_then(Value::as_bool);
let upstream_code = metadata_string_field(metadata, "upstreamCode");
let failure_class = metadata_string_field(metadata, "failureClass");
let failure_source = metadata_string_field(metadata, "failureSource");
let failure_detail = metadata_string_field(metadata, "failureDetail");
macro_rules! emit_stream_finalized_log {
($level:expr) => {
tracing::event!(
$level,
account_id = %context.account_id,
request_id = %context.request_id,
route = %context.route,
model = %context.display_model,
status_code,
usage_level = ?level,
event_message = %message,
transport = %backend_transport_name(context.transport),
websocket_pool_kind = ?websocket_pool_kind,
websocket_pool_reason = ?websocket_pool_reason,
response_id = response_id.as_deref().unwrap_or(""),
first_token_ms = ?first_token_ms,
latency_ms = elapsed_millis_i64(context.started_at),
completed = ?completed,
failed = ?failed,
upstream_code = ?upstream_code,
failure_class = ?failure_class,
failure_source = ?failure_source,
failure_detail = ?failure_detail,
"live response stream finalized"
);
};
}
match level {
UsageRecordLevel::Debug => {
emit_stream_finalized_log!(tracing::Level::DEBUG);
}
UsageRecordLevel::Info => {
emit_stream_finalized_log!(tracing::Level::INFO);
}
UsageRecordLevel::Warn => {
emit_stream_finalized_log!(tracing::Level::WARN);
}
UsageRecordLevel::Error => {
emit_stream_finalized_log!(tracing::Level::ERROR);
}
}
}
fn metadata_string_field<'a>(metadata: &'a Value, field: &str) -> Option<&'a str> {
metadata.get(field).and_then(Value::as_str)
}
pub(super) async fn live_response_rate_limit_headers(
context: &LiveResponseStreamContext,
) -> Vec<(String, String)> {
let mut headers = context.rate_limit_headers.clone();
if let Some(updates) = &context.rate_limit_header_updates {
headers.extend(updates.lock().await.iter().cloned());
}
headers
}
pub(super) async fn live_response_turn_state(
context: &LiveResponseStreamContext,
) -> Option<String> {
if let Some(update) = &context.turn_state_update {
return update.lock().await.clone();
}
context.turn_state.clone()
}
pub(super) fn insert_first_token_ms(metadata: &mut Value, first_token_ms: Option<i64>) {
let Some(first_token_ms) = first_token_ms else {
return;
};
if let Some(object) = metadata.as_object_mut() {
object.insert(
"firstTokenMs".to_string(),
Value::Number(first_token_ms.into()),
);
}
}
pub(super) fn insert_websocket_pool_decision(
metadata: &mut Value,
decision: Option<WebSocketPoolDecision>,
) {
let Some(decision) = decision else {
return;
};
if let Some(object) = metadata.as_object_mut() {
object.insert("websocketPool".to_string(), decision.metadata_value());
}
}
@@ -0,0 +1,143 @@
//! 可换号上游失败的账号风险剥离。
use chrono::Utc;
use crate::{
dispatch::{
errors::{
is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error,
is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body,
upstream_error_http_status,
},
recovery::{
auth::{auth_failure_account_status, is_auth_upstream_error},
cloudflare::{
cloudflare_challenge_error_message, cloudflare_path_block_error_message,
is_cloudflare_challenge_upstream_error, is_cloudflare_path_block_upstream_error,
CloudflareRecovery,
},
exhaustion::AccountExhaustionTracker,
},
stream::sse_failure::{
auth_sse_failure_account_status, is_auth_sse_failure, is_model_unsupported_sse_failure,
is_quota_exhausted_sse_failure, sse_failure_error_body, stream_failure_http_status,
},
},
fleet::{account::AccountStatus, pool::AccountPoolService},
upstream::openai::{
protocol::responses::ResponsesSseFailure,
transport::{is_banned_upstream_error, CodexClientError},
},
};
/// 处理需要隔离当前账号并交回调度器选择下一候选的上游失败。
///
/// 本函数不选择下一个账号,也不实现固定轮换策略;调用方继续使用请求级候选账本,
/// 因而始终服从当前运行时调度策略。
pub(crate) async fn isolate_rotatable_account_failure(
account_pool: &AccountPoolService,
cloudflare: &CloudflareRecovery,
exhausted_accounts: &mut AccountExhaustionTracker,
account_id: &str,
error: &CodexClientError,
) -> bool {
if is_rate_limit_upstream_error(error) {
exhausted_accounts.record_rate_limited(Some(account_id), upstream_error_body(error));
account_pool
.mark_quota_limited_until(account_id, rate_limit_cooldown_until(error, Utc::now()))
.await;
return true;
}
if is_quota_exhausted_upstream_error(error) {
exhausted_accounts.record_quota_exhausted(Some(account_id), upstream_error_body(error));
account_pool
.set_status(account_id, AccountStatus::QuotaExhausted)
.await;
return true;
}
if is_auth_upstream_error(error) {
let account_status = auth_failure_account_status(error);
exhausted_accounts.record_auth_failure(
Some(account_id),
account_status,
upstream_error_body(error),
Some(upstream_error_http_status(error)),
);
account_pool.set_status(account_id, account_status).await;
return true;
}
if is_cloudflare_challenge_upstream_error(error) {
exhausted_accounts
.record_cloudflare_challenge(Some(account_id), cloudflare_challenge_error_message());
cloudflare.apply_challenge(account_pool, account_id).await;
return true;
}
if is_cloudflare_path_block_upstream_error(error) {
exhausted_accounts.record_cloudflare_path_blocked(
Some(account_id),
cloudflare_path_block_error_message(),
);
cloudflare.apply_path_block(account_pool, account_id).await;
return true;
}
if is_model_unsupported_upstream_error(error) {
exhausted_accounts.record_model_unsupported(Some(account_id), upstream_error_body(error));
return true;
}
if is_banned_upstream_error(error) {
exhausted_accounts.record_auth_failure(
Some(account_id),
AccountStatus::Banned,
upstream_error_body(error),
Some(upstream_error_http_status(error)),
);
account_pool
.set_status(account_id, AccountStatus::Banned)
.await;
return true;
}
false
}
/// 处理 SSE `response.failed` 中需要隔离当前账号的失败。
pub(crate) async fn isolate_sse_account_failure(
account_pool: &AccountPoolService,
exhausted_accounts: &mut AccountExhaustionTracker,
account_id: &str,
failure: &ResponsesSseFailure,
) -> bool {
if is_model_unsupported_sse_failure(failure) {
exhausted_accounts
.record_model_unsupported(Some(account_id), sse_failure_error_body(failure));
return true;
}
if is_quota_exhausted_sse_failure(failure) {
exhausted_accounts.record_quota_exhausted(Some(account_id), failure.message.clone());
account_pool
.set_status(account_id, AccountStatus::QuotaExhausted)
.await;
return true;
}
if is_auth_sse_failure(failure) {
let account_status = auth_sse_failure_account_status(failure);
exhausted_accounts.record_auth_failure(
Some(account_id),
account_status,
sse_failure_error_body(failure),
Some(stream_failure_http_status(failure)),
);
account_pool.set_status(account_id, account_status).await;
return true;
}
false
}

Some files were not shown because too many files have changed in this diff Show More