mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
71d87c5ab9 | ||
|
|
e23689ae22 | ||
|
|
4e527feb38 | ||
|
|
537c749b73 | ||
|
|
28b89200f7 | ||
|
|
a8f5c5a54e | ||
|
|
d94bca33dd | ||
|
|
9b33e60df2 | ||
|
|
1e99c5037d | ||
|
|
26cf170f33 | ||
|
|
8d54171771 | ||
|
|
f30652e113 | ||
|
|
021e87c59e | ||
|
|
f867f13ce7 | ||
|
|
d7bb5e4ab2 | ||
|
|
5d0df748c5 | ||
|
|
b3b5377683 | ||
|
|
203fc1b03e | ||
|
|
b8e5b6eade | ||
|
|
62df4d714c | ||
|
|
4dcd9dc31d | ||
|
|
49120d0efb | ||
|
|
4d48a7bca1 | ||
|
|
3c33e93da9 | ||
|
|
1652b60769 | ||
|
|
899454e6b5 | ||
|
|
7a6c6927bf | ||
|
|
7cb186c4ae | ||
|
|
fe5494a614 | ||
|
|
61cdaf753e | ||
|
|
f06530278d | ||
|
|
45e4b36225 | ||
|
|
68e76d8138 | ||
|
|
89ff4be281 | ||
|
|
788e587259 | ||
|
|
cbf9ce91a1 | ||
|
|
71e51a868f | ||
|
|
3c9ec50bf2 | ||
|
|
c1b81809a6 | ||
|
|
939f392438 | ||
|
|
ff0de4742d | ||
|
|
44f45101b8 | ||
|
|
a240668a47 | ||
|
|
30f354965c | ||
|
|
3b462f7ecf | ||
|
|
5682d4fbe9 | ||
|
|
146776c742 | ||
|
|
e8c14d4324 | ||
|
|
5483e37b00 | ||
|
|
daad7c746c | ||
|
|
6e24037c4d | ||
|
|
95daf7093e | ||
|
|
2e121c0670 | ||
|
|
66916ab8b2 | ||
|
|
2945523140 | ||
|
|
f375d2d77c | ||
|
|
c6e1b347be | ||
|
|
1ac9803c34 | ||
|
|
352b781340 | ||
|
|
62a6dfc489 | ||
|
|
e91f91b542 | ||
|
|
380556369b | ||
|
|
595cda541d | ||
|
|
90ed8bf6bc | ||
|
|
1101eee628 | ||
|
|
0e952c9d83 | ||
|
|
875d610812 | ||
|
|
9c37d262d1 | ||
|
|
bd95adadf9 | ||
|
|
6e6627b0c9 | ||
|
|
3579c28b75 | ||
|
|
2143de2791 | ||
|
|
298357f97c |
@@ -0,0 +1,54 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: monday
|
||||
time: '09:00'
|
||||
timezone: Asia/Shanghai
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
- minor
|
||||
- patch
|
||||
|
||||
- package-ecosystem: cargo
|
||||
directory: /backend
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: monday
|
||||
time: '09:10'
|
||||
timezone: Asia/Shanghai
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
- minor
|
||||
- patch
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /frontend
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: monday
|
||||
time: '09:20'
|
||||
timezone: Asia/Shanghai
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
- minor
|
||||
- patch
|
||||
|
||||
- package-ecosystem: docker
|
||||
directory: /deploy
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: monday
|
||||
time: '09:30'
|
||||
timezone: Asia/Shanghai
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
- minor
|
||||
- patch
|
||||
@@ -0,0 +1,248 @@
|
||||
name: Reusable Container
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: Exact commit SHA to build and validate
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
container:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: cpr-ci-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
CPR_IMAGE: codex-proxy-rs:ci-${{ inputs.ref }}
|
||||
CPR_ADMIN_DEFAULT_PASSWORD: 'admin @:/?#%2F ci password'
|
||||
CPR_POSTGRES_PASSWORD: 'pg @:/?#%2F ci'
|
||||
CPR_REDIS_PASSWORD: 'redis @:/?#%2F ci'
|
||||
CPR_SMOKE_PORT: '18080'
|
||||
TRIVY_QUIET: 'true'
|
||||
TRIVY_SKIP_VERSION_CHECK: 'true'
|
||||
steps:
|
||||
- name: Define isolated runtime paths
|
||||
shell: bash
|
||||
run: |
|
||||
{
|
||||
echo "CPR_CONFIG_FILE=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/config.yaml"
|
||||
echo "CPR_DATA_DIR=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/data"
|
||||
echo "CPR_LOG_DIR=$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT/logs"
|
||||
echo "DIAGNOSTICS_DIR=$RUNNER_TEMP/codex-proxy-rs-diagnostics"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: deploy/Dockerfile
|
||||
target: runtime
|
||||
load: true
|
||||
tags: ${{ env.CPR_IMAGE }}
|
||||
cache-from: type=gha,scope=runtime-ci
|
||||
cache-to: type=gha,mode=max,scope=runtime-ci
|
||||
|
||||
- name: Generate Trivy SARIF
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.CPR_IMAGE }}
|
||||
version: v0.72.0
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy SARIF
|
||||
if: >-
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
- name: Enforce Trivy vulnerability gate
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.CPR_IMAGE }}
|
||||
version: v0.72.0
|
||||
format: table
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '1'
|
||||
|
||||
- name: Prepare isolated runtime
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
runtime_dir="$(dirname "$CPR_CONFIG_FILE")"
|
||||
mkdir -p "$runtime_dir" "$DIAGNOSTICS_DIR"
|
||||
sudo install -d -m 0770 -o "$(id -u)" -g 10001 "$CPR_DATA_DIR" "$CPR_LOG_DIR"
|
||||
cp deploy/config.example.yaml "$CPR_CONFIG_FILE"
|
||||
sudo chown "$(id -u):10001" "$CPR_CONFIG_FILE"
|
||||
chmod 0640 "$CPR_CONFIG_FILE"
|
||||
|
||||
- name: Validate Docker Compose
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml config --quiet
|
||||
|
||||
- name: Configure Redis host memory overcommit
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo sysctl -w vm.overcommit_memory=1
|
||||
test "$(sysctl -n vm.overcommit_memory)" = 1
|
||||
|
||||
- name: Verify PostgreSQL and Redis volume persistence
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
|
||||
"${compose[@]}" up -d --no-build --wait postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c "SHOW TimeZone;" | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"CREATE TABLE cpr_ci_persistence_probe (value text PRIMARY KEY); INSERT INTO cpr_ci_persistence_probe VALUES ('persisted');"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
|
||||
redis-cli SET cpr:ci:persistence-probe persisted
|
||||
"${compose[@]}" up -d --no-build --wait --force-recreate postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c \
|
||||
"SELECT value FROM cpr_ci_persistence_probe;")" = persisted
|
||||
test "$("${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
|
||||
redis-cli --raw GET cpr:ci:persistence-probe | tr -d '\r')" = persisted
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CPR_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"DROP TABLE cpr_ci_persistence_probe;"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CPR_REDIS_PASSWORD" redis \
|
||||
redis-cli DEL cpr:ci:persistence-probe
|
||||
|
||||
- name: Start and verify application
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
|
||||
"${compose[@]}" up -d --no-build --wait codex-proxy-rs
|
||||
status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
"http://127.0.0.1:${CPR_SMOKE_PORT}/healthz")"
|
||||
test "$status" = 204
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -u | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -g | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs date +%z | tr -d '\r')" = +0800
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs cat /etc/timezone | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T codex-proxy-rs sh -c \
|
||||
'touch /app/data/.ci-write /app/logs/.ci-write && rm /app/data/.ci-write /app/logs/.ci-write'
|
||||
|
||||
- name: Verify clean logs and graceful shutdown
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
|
||||
"${compose[@]}" stop --timeout 30 codex-proxy-rs
|
||||
container_id="$("${compose[@]}" ps -a -q codex-proxy-rs)"
|
||||
test -n "$container_id"
|
||||
test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = 0
|
||||
"${compose[@]}" logs --no-color --timestamps codex-proxy-rs \
|
||||
> "$DIAGNOSTICS_DIR/application.log"
|
||||
"${compose[@]}" logs --no-color --timestamps postgres \
|
||||
> "$DIAGNOSTICS_DIR/postgres.log"
|
||||
"${compose[@]}" logs --no-color --timestamps redis \
|
||||
> "$DIAGNOSTICS_DIR/redis.log"
|
||||
if grep --extended-regexp --ignore-case \
|
||||
'"level"[[:space:]]*:[[:space:]]*"(warn|error|fatal|panic)"|(^|[^[:alnum:]_])panic([^[:alnum:]_]|$)' \
|
||||
"$DIAGNOSTICS_DIR/application.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'(^|[[:space:]])(warning|error|fatal|panic):' \
|
||||
"$DIAGNOSTICS_DIR/postgres.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'[0-9]+:[A-Z][[:space:]].*[[:space:]]#[[:space:]]|(^|[|[:space:]])(warning|error|fatal|panic)(:|[[:space:]])' \
|
||||
"$DIAGNOSTICS_DIR/redis.log"; then
|
||||
echo "container logs contain a warning or error severity" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Collect container diagnostics
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set +e
|
||||
mkdir -p "$DIAGNOSTICS_DIR"
|
||||
raw_dir="$(mktemp -d)"
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml)
|
||||
"${compose[@]}" config > "$raw_dir/compose-config.yaml" 2>&1
|
||||
"${compose[@]}" ps -a > "$raw_dir/compose-ps.txt" 2>&1
|
||||
"${compose[@]}" logs --no-color --timestamps > "$raw_dir/compose.log" 2>&1
|
||||
mapfile -t container_ids < <("${compose[@]}" ps -a -q)
|
||||
if [ "${#container_ids[@]}" -gt 0 ]; then
|
||||
docker inspect "${container_ids[@]}" > "$raw_dir/docker-inspect.json" 2>&1
|
||||
fi
|
||||
RAW_DIR="$raw_dir" OUTPUT_DIR="$DIAGNOSTICS_DIR" ruby <<'RUBY'
|
||||
raw_dir = ENV.fetch("RAW_DIR")
|
||||
output_dir = ENV.fetch("OUTPUT_DIR")
|
||||
secrets = [
|
||||
ENV.fetch("CPR_ADMIN_DEFAULT_PASSWORD"),
|
||||
ENV.fetch("CPR_POSTGRES_PASSWORD"),
|
||||
ENV.fetch("CPR_REDIS_PASSWORD"),
|
||||
]
|
||||
encoded_secrets = secrets.map do |secret|
|
||||
secret.encode("UTF-8").bytes.map do |byte|
|
||||
character = byte.chr
|
||||
character.match?(/[A-Za-z0-9]/) ? character : format("%%%02X", byte)
|
||||
end.join
|
||||
end
|
||||
|
||||
Dir.children(raw_dir).each do |name|
|
||||
content = File.binread(File.join(raw_dir, name))
|
||||
(secrets + encoded_secrets).each do |secret|
|
||||
content = content.gsub(secret, "<redacted>")
|
||||
end
|
||||
File.binwrite(File.join(output_dir, name), content)
|
||||
end
|
||||
RUBY
|
||||
rm -rf "$raw_dir"
|
||||
cp trivy-results.sarif "$DIAGNOSTICS_DIR/" 2>/dev/null || true
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Remove isolated runtime
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/docker-compose.yml -f deploy/docker-compose.smoke.yml \
|
||||
down --volumes --remove-orphans || true
|
||||
sudo rm -rf "$GITHUB_WORKSPACE/.runtime/ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
||||
@@ -0,0 +1,136 @@
|
||||
name: Reusable Quality
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: Exact commit SHA to validate
|
||||
required: true
|
||||
type: string
|
||||
upload_frontend:
|
||||
description: Upload the built frontend for release jobs
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CPR_TEST_DATABASE_URL: postgres://codex_proxy:codex_proxy@127.0.0.1:5432/codex_proxy
|
||||
CPR_TEST_REDIS_URL: redis://127.0.0.1:6379
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18-bookworm@sha256:d9c83446333daec3f0588cc709adb80c26090b7f9f0f7ec8d43c243385d79818
|
||||
env:
|
||||
POSTGRES_DB: codex_proxy
|
||||
POSTGRES_USER: codex_proxy
|
||||
POSTGRES_PASSWORD: codex_proxy
|
||||
POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U codex_proxy -d codex_proxy"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:8-alpine@sha256:9d317178eceac8454a2284a9e6df2466b93c745529947f0cd42a0fa9609d7005
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
with:
|
||||
components: rustfmt,clippy
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
- name: Rust format
|
||||
working-directory: backend
|
||||
run: cargo fmt --check
|
||||
|
||||
- name: Rust clippy
|
||||
working-directory: backend
|
||||
run: cargo clippy --all-targets --all-features --locked
|
||||
|
||||
- name: Rust tests
|
||||
working-directory: backend
|
||||
run: cargo test --test main --locked
|
||||
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
CI: 'true'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install frontend dependencies
|
||||
run: pnpm --dir frontend install --frozen-lockfile
|
||||
|
||||
- name: Frontend format
|
||||
run: pnpm --dir frontend format:check
|
||||
|
||||
- name: Frontend build
|
||||
run: pnpm --dir frontend build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: inputs.upload_frontend
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
workflow-lint:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Lint GitHub Actions workflows
|
||||
run: >-
|
||||
docker run --rm
|
||||
--volume "$PWD:/repo"
|
||||
--workdir /repo
|
||||
rhysd/actionlint:1.7.7@sha256:887a259a5a534f3c4f36cb02dca341673c6089431057242cdc931e9f133147e9
|
||||
-color
|
||||
.github/workflows/*.yml
|
||||
+33
-59
@@ -10,73 +10,47 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.workflow }}-${{ github.ref }}
|
||||
group: ci-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
container: ${{ steps.filter.outputs.container }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1.95
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
components: rustfmt,clippy
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@v2.9.1
|
||||
- name: Detect container changes
|
||||
id: filter
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
filters: |
|
||||
container:
|
||||
- 'backend/**'
|
||||
- 'frontend/**'
|
||||
- 'deploy/**'
|
||||
- '.dockerignore'
|
||||
- 'release/version.yaml'
|
||||
- '.github/workflows/_container.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
|
||||
- name: Rust format
|
||||
working-directory: backend
|
||||
run: cargo fmt --check
|
||||
quality:
|
||||
uses: ./.github/workflows/_quality.yml
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Rust clippy
|
||||
working-directory: backend
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Rust tests
|
||||
working-directory: backend
|
||||
run: cargo test --test main --locked
|
||||
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install frontend dependencies
|
||||
run: pnpm --dir frontend install --frozen-lockfile
|
||||
|
||||
- name: Frontend format
|
||||
run: pnpm --dir frontend format:check
|
||||
|
||||
- name: Frontend build
|
||||
run: pnpm --dir frontend build
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Validate Docker Compose
|
||||
run: docker compose -f deploy/docker-compose.yml config
|
||||
|
||||
- name: Build runtime image
|
||||
run: docker build -f deploy/Dockerfile --target runtime -t codex-proxy-rs:ci .
|
||||
container:
|
||||
needs: [changes, quality]
|
||||
if: needs.changes.outputs.container == 'true'
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
uses: ./.github/workflows/_container.yml
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
@@ -25,6 +25,7 @@ env:
|
||||
jobs:
|
||||
release-info:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
@@ -35,10 +36,11 @@ jobs:
|
||||
non_docker_matrix: ${{ steps.version.outputs.non_docker_matrix }}
|
||||
docker_platforms: ${{ steps.version.outputs.docker_platforms }}
|
||||
previous_tag: ${{ steps.version.outputs.previous_tag }}
|
||||
prerelease: ${{ steps.version.outputs.prerelease }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
fetch-depth: 0
|
||||
@@ -151,62 +153,33 @@ jobs:
|
||||
puts "non_docker_matrix=#{non_docker_matrix}"
|
||||
puts "docker_platforms=#{docker_platforms.join(",")}"
|
||||
puts "previous_tag=#{previous_tag}"
|
||||
puts "prerelease=#{version.split("+", 2).first.include?("-")}"
|
||||
RUBY
|
||||
|
||||
build-frontend:
|
||||
quality:
|
||||
needs: [release-info]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: frontend/pnpm-lock.yaml
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: frontend
|
||||
|
||||
- name: Build frontend
|
||||
run: pnpm build
|
||||
working-directory: frontend
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
uses: ./.github/workflows/_quality.yml
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
upload_frontend: true
|
||||
|
||||
build-docker-binary:
|
||||
name: build-docker-binary (${{ matrix.os }}-${{ matrix.arch }})
|
||||
needs: [release-info]
|
||||
needs: [release-info, quality]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.release-info.outputs.docker_matrix) }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -232,9 +205,8 @@ jobs:
|
||||
-v "$HOME/.cargo:/cargo" \
|
||||
-v "$PWD/..:/workspace" \
|
||||
-w /workspace/backend \
|
||||
rust:1.95-bookworm \
|
||||
rust:1.97-bookworm@sha256:7d0723df719e7f213b69dc7c8c595985c3f4b060cfbee4f7bc0e347a86fe3b6a \
|
||||
cargo build --release --locked --target "${{ matrix.target }}" --bin codex-proxy-rs
|
||||
mkdir -p "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}"
|
||||
cp "target/${{ matrix.target }}/release/${{ matrix.binary }}" "../dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/${{ matrix.binary }}"
|
||||
working-directory: backend
|
||||
env:
|
||||
@@ -244,7 +216,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: release
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -253,20 +225,26 @@ jobs:
|
||||
|
||||
build-binary:
|
||||
name: build-binary (${{ matrix.os }}-${{ matrix.arch }})
|
||||
needs: [release-info]
|
||||
needs: [release-info, quality]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.release-info.outputs.non_docker_matrix) }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache Cargo
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -276,10 +254,6 @@ jobs:
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Install Rust target
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
working-directory: backend
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -294,7 +268,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: release
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -302,8 +276,9 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
package-assets:
|
||||
needs: [release-info, build-frontend, build-docker-binary, build-binary]
|
||||
needs: [release-info, quality, build-docker-binary, build-binary]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
@@ -311,18 +286,18 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/frontend
|
||||
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: codex-proxy-rs-*
|
||||
path: dist/binaries
|
||||
@@ -407,12 +382,12 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@v4
|
||||
uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
- name: Upload release assets
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
@@ -421,30 +396,34 @@ jobs:
|
||||
|
||||
build-docker-image:
|
||||
name: build-docker-image (${{ matrix.dockerPlatform }})
|
||||
needs: [release-info, build-frontend, build-docker-binary]
|
||||
needs: [release-info, quality, build-docker-binary]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.release-info.outputs.docker_matrix) }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
env:
|
||||
TRIVY_QUIET: 'true'
|
||||
TRIVY_SKIP_VERSION_CHECK: 'true'
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/docker/web/dist
|
||||
|
||||
- name: Download Docker binary artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
@@ -476,10 +455,10 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -487,7 +466,7 @@ jobs:
|
||||
|
||||
- name: Build and push app image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: deploy/Dockerfile
|
||||
@@ -505,13 +484,26 @@ jobs:
|
||||
org.opencontainers.image.version=${{ needs.release-info.outputs.version }}
|
||||
org.opencontainers.image.revision=${{ needs.release-info.outputs.git_sha }}
|
||||
org.opencontainers.image.source=https://github.com/zyycn/codex-proxy-rs
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}-${{ matrix.arch }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}-${{ matrix.arch }}
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}-${{ matrix.arch }}
|
||||
|
||||
- name: Scan exact app image digest
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
env:
|
||||
TRIVY_PLATFORM: ${{ matrix.dockerPlatform }}
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }}
|
||||
version: v0.72.0
|
||||
format: table
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '1'
|
||||
|
||||
publish:
|
||||
needs: [release-info, package-assets, build-docker-image]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
@@ -520,24 +512,24 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.tag }}
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download release assets
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Install Cosign
|
||||
uses: sigstore/cosign-installer@v4.1.2
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -550,6 +542,7 @@ jobs:
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
DOCKER_PLATFORMS: ${{ needs.release-info.outputs.docker_platforms }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -560,7 +553,7 @@ jobs:
|
||||
platform="${platform//[[:space:]]/}"
|
||||
[ -n "${platform}" ] || continue
|
||||
arch="${platform##*/}"
|
||||
sources+=("${image}:${VERSION}-${arch}")
|
||||
sources+=("${image}:sha-${GIT_SHA}-${arch}")
|
||||
done
|
||||
|
||||
if [ "${#sources[@]}" -eq 0 ]; then
|
||||
@@ -568,11 +561,14 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker buildx imagetools create \
|
||||
-t "${image}:${VERSION}" \
|
||||
-t "${image}:latest" \
|
||||
-t "${image}:sha-${GIT_SHA}" \
|
||||
"${sources[@]}"
|
||||
target_args=(
|
||||
-t "${image}:${VERSION}"
|
||||
-t "${image}:sha-${GIT_SHA}"
|
||||
)
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
target_args+=(-t "${image}:latest")
|
||||
fi
|
||||
docker buildx imagetools create "${target_args[@]}" "${sources[@]}"
|
||||
|
||||
inspect_output="$(docker buildx imagetools inspect "${image}:${VERSION}")"
|
||||
platform_list="$(awk '$1 == "Platform:" { print $2 }' <<< "${inspect_output}")"
|
||||
@@ -592,7 +588,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@v4
|
||||
uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
@@ -602,10 +598,9 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
DIGEST: ${{ steps.manifest.outputs.digest }}
|
||||
IMAGE_TAGS: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.version }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}
|
||||
VERSION: ${{ needs.release-info.outputs.version }}
|
||||
GIT_SHA: ${{ needs.release-info.outputs.git_sha }}
|
||||
PRERELEASE: ${{ needs.release-info.outputs.prerelease }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${DIGEST}" ]; then
|
||||
@@ -613,20 +608,26 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
image="${REGISTRY}/${IMAGE_NAME}"
|
||||
tags=("${image}:${VERSION}" "${image}:sha-${GIT_SHA}")
|
||||
if [ "$PRERELEASE" != true ]; then
|
||||
tags+=("${image}:latest")
|
||||
fi
|
||||
images=()
|
||||
while IFS= read -r tag; do
|
||||
[ -n "${tag}" ] || continue
|
||||
for tag in "${tags[@]}"; do
|
||||
images+=("${tag}@${DIGEST}")
|
||||
done <<< "${IMAGE_TAGS}"
|
||||
done
|
||||
|
||||
cosign sign --yes "${images[@]}"
|
||||
|
||||
- name: Publish GitHub Release
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
||||
with:
|
||||
tag_name: ${{ needs.release-info.outputs.tag }}
|
||||
generate_release_notes: true
|
||||
previous_tag: ${{ needs.release-info.outputs.previous_tag }}
|
||||
prerelease: ${{ needs.release-info.outputs.prerelease == 'true' }}
|
||||
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
|
||||
body: |
|
||||
## 📥 Installation
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: security-scan-${{ github.workflow }}-${{ github.ref }}
|
||||
group: security-scan-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
@@ -21,18 +21,20 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1.95
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@v2.9.1
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
- name: Install cargo-audit
|
||||
run: cargo install cargo-audit --locked
|
||||
run: cargo install cargo-audit --version 0.22.2 --locked
|
||||
|
||||
- name: Run cargo audit
|
||||
working-directory: backend
|
||||
@@ -43,15 +45,17 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
@@ -63,21 +67,11 @@ jobs:
|
||||
- name: Run pnpm audit
|
||||
run: pnpm --dir frontend audit --prod --audit-level=high
|
||||
|
||||
docker-security:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build runtime image
|
||||
run: docker build -f deploy/Dockerfile --target runtime -t codex-proxy-rs:security-scan .
|
||||
|
||||
- name: Scan runtime image
|
||||
uses: aquasecurity/trivy-action@v0.36.0
|
||||
with:
|
||||
image-ref: codex-proxy-rs:security-scan
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
ignore-unfixed: true
|
||||
exit-code: '1'
|
||||
container-security:
|
||||
if: github.event_name == 'schedule'
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
uses: ./.github/workflows/_container.yml
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
+34
-29
@@ -1,42 +1,47 @@
|
||||
# Rust build output.
|
||||
# Rust
|
||||
/backend/target/
|
||||
/target/
|
||||
/dist/
|
||||
/package/
|
||||
/checksums.txt
|
||||
/codex-proxy-rs_*.tar.gz
|
||||
/codex-proxy-rs_*.zip
|
||||
*.profraw
|
||||
*.profdata
|
||||
|
||||
# Frontend dependencies and build output.
|
||||
# Frontend
|
||||
/frontend/node_modules/
|
||||
/frontend/dist/
|
||||
/frontend/.vite/
|
||||
/.pnpm-store/
|
||||
/frontend/.pnpm-store/
|
||||
*.tsbuildinfo
|
||||
|
||||
# Application runtime state.
|
||||
*.runtime
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
# Release artifacts
|
||||
/dist/
|
||||
|
||||
# Runtime data
|
||||
/.runtime/
|
||||
*.log
|
||||
|
||||
# Local configuration.
|
||||
/local.yaml
|
||||
/local.yml
|
||||
/.env
|
||||
# Local configuration and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!**/.env.example
|
||||
/.envrc
|
||||
/config.yaml
|
||||
/backend/config.yaml
|
||||
/deploy/.env
|
||||
|
||||
# Local workspaces and tooling state.
|
||||
# Editors and operating systems
|
||||
/.idea/
|
||||
/.vscode/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Local tools and worktrees
|
||||
/.codegraph/
|
||||
/.codex-ws-audit/
|
||||
/.agents/
|
||||
/.reasonix/
|
||||
/.worktrees/
|
||||
/worktrees/
|
||||
/.codex-ws-audit/
|
||||
/.codegraph/
|
||||
*.tmp
|
||||
*.swp
|
||||
*~
|
||||
*.codegraph
|
||||
/.agents/
|
||||
/.cargo/
|
||||
/.reasonix/
|
||||
*.tmp
|
||||
|
||||
@@ -1,204 +1,225 @@
|
||||
<h1 align="center">Codex Proxy RS</h1>
|
||||
<div align="center">
|
||||
<h1>Codex Proxy RS</h1>
|
||||
<p>OpenAI Responses 兼容的 Codex 多账号网关</p>
|
||||
<p>
|
||||
<a href="https://www.rust-lang.org/"><img alt="Rust 1.97" src="https://img.shields.io/badge/Rust-1.97-000?style=flat-square&logo=rust" /></a>
|
||||
<a href="https://github.com/tokio-rs/axum"><img alt="Axum 0.8" src="https://img.shields.io/badge/Axum-0.8-2f7d95?style=flat-square" /></a>
|
||||
<a href="https://vuejs.org/"><img alt="Vue 3.5" src="https://img.shields.io/badge/Vue-3.5-42b883?style=flat-square&logo=vuedotjs&logoColor=white" /></a>
|
||||
<a href="https://vite.dev/"><img alt="Vite 8" src="https://img.shields.io/badge/Vite-8-646cff?style=flat-square&logo=vite&logoColor=white" /></a>
|
||||
<a href="https://www.postgresql.org/"><img alt="PostgreSQL 18" src="https://img.shields.io/badge/PostgreSQL-18-4169e1?style=flat-square&logo=postgresql&logoColor=white" /></a>
|
||||
<a href="https://redis.io/"><img alt="Redis 8" src="https://img.shields.io/badge/Redis-8-ff4438?style=flat-square&logo=redis&logoColor=white" /></a>
|
||||
<a href="https://opensource.org/license/mit"><img alt="MIT License" src="https://img.shields.io/badge/License-MIT-blue?style=flat-square" /></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="#快速部署">快速部署</a>
|
||||
·
|
||||
<a href="#客户端接入">客户端接入</a>
|
||||
·
|
||||
<a href="docs/architecture.md">架构说明</a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<p align="center">
|
||||
OpenAI 兼容的 ChatGPT / Codex 多账号代理网关
|
||||
</p>
|
||||
Codex Proxy RS 为多个 Codex 账号提供统一的 Responses API。客户端使用同一个服务地址和 API Key;账号调度、并发限制、额度检查、故障换号和会话续接都在服务端完成。
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/Rust-1.95-000?style=flat-square&logo=rust" />
|
||||
<img src="https://img.shields.io/badge/Axum-0.8-2f7d95?style=flat-square" />
|
||||
<img src="https://img.shields.io/badge/Vue-3.5-42b883?style=flat-square&logo=vuedotjs&logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/Vite-8-646cff?style=flat-square&logo=vite&logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/SQLite-local-003b57?style=flat-square&logo=sqlite" />
|
||||
<img src="https://img.shields.io/badge/License-MIT-blue?style=flat-square" />
|
||||
</p>
|
||||
## 功能
|
||||
|
||||
## 概览
|
||||
- 兼容 `/v1/responses`、SSE、官方 Responses WebSocket 和模型查询。
|
||||
- 客户端与上游均可使用 HTTP/SSE 或 WebSocket。
|
||||
- 提供 `smart`、额度重置优先、轮询和粘性四种账号调度策略。
|
||||
- 单个账号发生额度、认证、模型或传输故障时,继续使用账号池中的其他可用账号。
|
||||
- 支持跨账号会话续接,并隔离不同账号的会话状态。
|
||||
- 提供账号、API Key、模型映射、运行参数、用量、错误记录和系统更新管理界面。
|
||||
- 使用 PostgreSQL 保存权威业务数据,使用 Redis 保存管理会话、租约、模型快照和响应归属。
|
||||
- 内置日志轮转、遥测保留期清理和 PostgreSQL/Redis 健康检查。
|
||||
|
||||
Codex Proxy RS 是一个单进程服务:
|
||||
除账号调度和会话恢复外,请求与响应尽量保持原有语义。
|
||||
|
||||
- Rust/Axum 后端提供 OpenAI 兼容 `/v1/*` 代理、管理端 API、SQLite 持久化和静态前端托管。
|
||||
- Vue 管理端用于账号导入、API Key、用量统计、请求记录、运行参数、模型映射和系统更新。
|
||||
- 运行数据默认收敛到仓库 `.runtime/`,Docker 也使用同一套宿主机目录做 bind mount。
|
||||
## 快速部署
|
||||
|
||||
## 环境
|
||||
需要 Docker Engine 和 Docker Compose Plugin。下面的 Compose 默认使用:
|
||||
|
||||
- Rust 1.95
|
||||
- Node 24 或兼容版本,pnpm 11(只在开发前端或构建镜像时需要)
|
||||
- Docker / Docker Compose(部署或验证镜像时需要)
|
||||
- 应用镜像:`ghcr.io/zyycn/codex-proxy-rs:latest`
|
||||
- PostgreSQL 18
|
||||
- Redis 8
|
||||
- 管理端地址:`http://127.0.0.1:8080`
|
||||
|
||||
## 本地运行
|
||||
|
||||
本地直接运行后端时,配置文件由 `CPR_CONFIG_FILE` 指定;未指定时读取当前工作目录下的 `config.yaml`。
|
||||
### 1. 获取项目
|
||||
|
||||
```bash
|
||||
cp deploy/config.example.yaml .runtime/config.yaml
|
||||
git clone https://github.com/zyycn/codex-proxy-rs.git
|
||||
cd codex-proxy-rs
|
||||
```
|
||||
|
||||
如果直接用本地二进制而不是 Docker,确保 `.runtime/config.yaml` 里的路径是宿主机路径:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
url: 'sqlite://.runtime/data/codex-proxy-rs.sqlite'
|
||||
|
||||
logging:
|
||||
directory: .runtime/logs
|
||||
|
||||
admin:
|
||||
default_password: '<set-a-long-random-password>'
|
||||
```
|
||||
|
||||
启动:
|
||||
|
||||
```bash
|
||||
mkdir -p .runtime/data .runtime/logs
|
||||
CPR_CONFIG_FILE=.runtime/config.yaml cargo run --manifest-path backend/Cargo.toml
|
||||
```
|
||||
|
||||
服务默认监听 `http://0.0.0.0:8080`。
|
||||
|
||||
## Docker 部署
|
||||
|
||||
Compose 默认把宿主机 `.runtime` 映射进容器:
|
||||
|
||||
- `.runtime/config.yaml` -> `/app/config.yaml`
|
||||
- `.runtime/data` -> `/app/data`
|
||||
- `.runtime/logs` -> `/app/logs`
|
||||
|
||||
初始化配置:
|
||||
### 2. 创建配置
|
||||
|
||||
```bash
|
||||
mkdir -p .runtime/data .runtime/logs
|
||||
cp deploy/config.example.yaml .runtime/config.yaml
|
||||
# 编辑 .runtime/config.yaml,设置 admin.default_password
|
||||
cp deploy/.env.example deploy/.env
|
||||
chmod 0600 deploy/.env
|
||||
```
|
||||
|
||||
Docker 配置文件里路径保持容器路径:
|
||||
编辑 `deploy/.env`:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
url: 'sqlite:///app/data/codex-proxy-rs.sqlite'
|
||||
|
||||
logging:
|
||||
directory: /app/logs
|
||||
```dotenv
|
||||
CPR_ADMIN_DEFAULT_PASSWORD='<管理员密码,至少 12 个字符>'
|
||||
CPR_POSTGRES_PASSWORD='<PostgreSQL 密码>'
|
||||
CPR_REDIS_PASSWORD='<Redis 密码>'
|
||||
```
|
||||
|
||||
构建并启动:
|
||||
`deploy/.env` 已被 Git 忽略。应用会安全地把 PostgreSQL 和 Redis 密码写入运行时连接 URL,不需要把真实密码填入 `.runtime/config.yaml`。
|
||||
|
||||
Linux 宿主机需要保证容器用户 `10001:10001` 可以读写数据和日志目录:
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml build
|
||||
docker compose -f deploy/docker-compose.yml up -d
|
||||
sudo chown -R "$(id -u):10001" .runtime/data .runtime/logs
|
||||
chmod 0770 .runtime/data .runtime/logs
|
||||
sudo chown "$(id -u):10001" .runtime/config.yaml
|
||||
chmod 0640 .runtime/config.yaml
|
||||
```
|
||||
|
||||
默认只绑定宿主机 `127.0.0.1:8080`。如需覆盖路径:
|
||||
### 3. 启动
|
||||
|
||||
```bash
|
||||
CPR_CONFIG_FILE=/path/to/config.yaml \
|
||||
CPR_DATA_DIR=/path/to/data \
|
||||
CPR_LOG_DIR=/path/to/logs \
|
||||
docker compose -f deploy/docker-compose.yml up -d
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml config
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml pull
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d --no-build
|
||||
```
|
||||
|
||||
## API
|
||||
检查服务:
|
||||
|
||||
客户端 API Key 在管理端创建。OpenAI 兼容接口通过 `Authorization: Bearer <key>` 鉴权。
|
||||
```bash
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml ps
|
||||
curl -i http://127.0.0.1:8080/healthz
|
||||
```
|
||||
|
||||
| 端点 | 说明 |
|
||||
| --- | --- |
|
||||
| `POST /v1/responses` | OpenAI Responses 兼容接口 |
|
||||
| `GET /v1/models` | 模型列表 |
|
||||
| `GET /v1/models/{id}` | 模型详情 |
|
||||
| `GET /v1/models/catalog` | 管理端可见模型目录 |
|
||||
| `/api/admin/*` | 管理端 API |
|
||||
`/healthz` 正常时返回 `204 No Content`。随后打开 <http://127.0.0.1:8080>:
|
||||
|
||||
示例:
|
||||
- 用户名:`admin@cpr.local`
|
||||
- 密码:`deploy/.env` 中的 `CPR_ADMIN_DEFAULT_PASSWORD`
|
||||
|
||||
首次登录后导入账号,在“API 密钥”页面创建客户端 Key。
|
||||
|
||||
### 本地构建镜像
|
||||
|
||||
需要验证当前源码时,不拉取 GHCR 镜像,直接构建:
|
||||
|
||||
```bash
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml build codex-proxy-rs
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
## 客户端接入
|
||||
|
||||
OpenAI 兼容客户端填写:
|
||||
|
||||
- Base URL:`http://127.0.0.1:8080/v1`
|
||||
- API Key:管理端创建的 `sk_...` 客户端 Key
|
||||
|
||||
Responses 请求示例:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8080/v1/responses \
|
||||
-H 'Authorization: Bearer <your-api-key>' \
|
||||
-H 'Authorization: Bearer <client-api-key>' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"model":"gpt-5.5","input":"Say hello","stream":true}'
|
||||
-d '{
|
||||
"model": "<model-id>",
|
||||
"input": "Reply with pong.",
|
||||
"stream": false
|
||||
}'
|
||||
```
|
||||
|
||||
可用接口:
|
||||
|
||||
| 路由 | 说明 |
|
||||
| --- | --- |
|
||||
| `POST /v1/responses` | Responses 非流式或 SSE 流式请求 |
|
||||
| `GET /v1/responses` | 官方 Responses WebSocket |
|
||||
| `POST /v1/responses/review` | Review 请求 |
|
||||
| `GET /v1/models` | 模型列表 |
|
||||
| `GET /v1/models/catalog` | 模型目录 |
|
||||
| `GET /v1/models/{model_id}` | 模型详情 |
|
||||
| `GET /v1/models/{model_id}/info` | 模型运行信息 |
|
||||
|
||||
Compose 默认只监听宿主机 `127.0.0.1`。需要从其他设备访问时,应在前面配置 HTTPS 反向代理,不要直接暴露 PostgreSQL 和 Redis 端口。
|
||||
|
||||
## 管理端
|
||||
|
||||
生产环境前端由后端直接托管,不需要单独 Node 服务。开发前端时:
|
||||
管理端提供以下页面:
|
||||
|
||||
- Dashboard:请求趋势、服务状态、账号概览和最近请求。
|
||||
- 账号:导入、导出、OAuth、连接测试、quota 和 token 刷新。
|
||||
- API 密钥:创建、停用、删除和客户端接入信息。
|
||||
- 用量:请求记录、token、延迟、模型分布和运维错误。
|
||||
- 设置:模型别名、账号调度策略、单账号并发、请求间隔和刷新参数。
|
||||
- 系统:版本检查、更新、重启和回滚。
|
||||
|
||||
管理端登录会话保存在 Redis,并按 `admin.session_ttl_minutes` 自动过期。
|
||||
|
||||
## 配置
|
||||
|
||||
启动配置位于 `.runtime/config.yaml`,模板见 [deploy/config.example.yaml](deploy/config.example.yaml)。
|
||||
|
||||
| 配置段 | 用途 |
|
||||
| --- | --- |
|
||||
| `server` | HTTP 监听地址和端口 |
|
||||
| `api` | Codex 上游地址 |
|
||||
| `database`、`redis` | PostgreSQL 与 Redis 连接 |
|
||||
| `quota` | quota 刷新周期和耗尽账号过滤 |
|
||||
| `tls` | 上游 HTTP 协议偏好 |
|
||||
| `ws_pool` | WebSocket 连接池与首事件超时 |
|
||||
| `fingerprint` | Codex Desktop 请求指纹 |
|
||||
| `admin` | 默认管理员和登录会话 TTL |
|
||||
| `logging` | 日志级别、目录、大小和保留期 |
|
||||
| `telemetry` | 请求事实记录开关 |
|
||||
|
||||
模型别名、调度策略、单账号并发、请求间隔和刷新参数由管理端保存到 PostgreSQL,可在运行中更新。请求事实、运维错误和聚合时间桶由后台任务按各自保留期自动清理。
|
||||
|
||||
## 数据与升级
|
||||
|
||||
| 数据 | 默认位置 |
|
||||
| --- | --- |
|
||||
| 账号、API Key、设置、用量和错误记录 | Compose 命名卷 `postgres-data` |
|
||||
| 管理会话、租约、模型快照和响应归属 | Compose 命名卷 `redis-data` |
|
||||
| 身份派生密钥和更新状态 | `.runtime/data` |
|
||||
| 文件日志 | `.runtime/logs` |
|
||||
| 启动配置 | `.runtime/config.yaml` |
|
||||
| Docker 密钥 | `deploy/.env` |
|
||||
|
||||
普通升级只替换应用镜像:
|
||||
|
||||
```bash
|
||||
pnpm --dir frontend install
|
||||
pnpm --dir frontend dev
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml pull codex-proxy-rs
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml up -d --no-build
|
||||
```
|
||||
|
||||
生产构建:
|
||||
更新镜像不会删除数据库卷,但仍应在升级前备份 PostgreSQL 和 `.runtime/data`。
|
||||
|
||||
不要使用 `docker compose down -v` 做普通升级,该命令会删除 PostgreSQL 与 Redis 命名卷。
|
||||
|
||||
`.runtime/data/identity_hmac_secret` 必须长期保留。丢失该文件会改变全部账号的 installation ID 和账号作用域身份。
|
||||
|
||||
## 查看状态
|
||||
|
||||
```bash
|
||||
pnpm --dir frontend build
|
||||
CPR_CONFIG_FILE=.runtime/config.yaml cargo run --manifest-path backend/Cargo.toml
|
||||
# 容器状态
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml ps
|
||||
|
||||
# 应用日志
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml logs -f codex-proxy-rs
|
||||
|
||||
# PostgreSQL 和 Redis 日志
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml logs postgres redis
|
||||
|
||||
# 展开并检查最终 Compose 配置
|
||||
docker compose --env-file deploy/.env -f deploy/docker-compose.yml config
|
||||
```
|
||||
|
||||
管理端主要功能:
|
||||
`/healthz` 返回 `503` 时,先检查 PostgreSQL 和 Redis 的健康状态。容器反复重启时,再检查 `.runtime/config.yaml`、`deploy/.env` 和 `.runtime/data`、`.runtime/logs` 的权限。
|
||||
|
||||
- 账号导入、OAuth 授权、连接测试、quota 刷新、token 刷新。
|
||||
- 客户端 API Key 管理。
|
||||
- 请求记录、Token 用量、模型用量和账号额度。
|
||||
- 运行参数、模型别名、账号选择策略。
|
||||
- 版本检查、在线更新、更新日志、重启和回滚。
|
||||
## 架构
|
||||
|
||||
## 配置与运行数据
|
||||
|
||||
配置文件只承载启动必需项。账号、API Key、模型映射、账号选择策略和多数运行参数由管理端写入 SQLite。
|
||||
|
||||
`.runtime` 约定:
|
||||
|
||||
```text
|
||||
.runtime/config.yaml # 本地或 Docker 启动配置,按实际运行环境写路径
|
||||
.runtime/data/codex-proxy-rs.sqlite # SQLite 数据库
|
||||
.runtime/data/installation_id # 上游 installation id
|
||||
.runtime/data/update-state.json # 在线更新状态
|
||||
.runtime/data/update.lock # 在线更新锁
|
||||
.runtime/logs/ # 日志
|
||||
```
|
||||
|
||||
## 发布与更新
|
||||
|
||||
版本信息在 `release/version.yaml`,发布平台在 `release/platforms.yaml`。
|
||||
|
||||
发布命令:
|
||||
|
||||
```bash
|
||||
release/publish 0.1.5
|
||||
```
|
||||
|
||||
该命令会更新版本文件、生成版本提交、创建 `v<version>` tag 并推送,随后 GitHub Actions 构建 Release 归档和 GHCR 多平台镜像。
|
||||
|
||||
在线更新由主服务处理,管理端调用 `/api/admin/system/*`:
|
||||
|
||||
- 检查 GitHub Release。
|
||||
- 下载匹配当前平台的归档和 `checksums.txt`。
|
||||
- 校验 checksum。
|
||||
- 替换二进制和 `web/dist`。
|
||||
- 重启时,Docker 模式依赖 `restart: unless-stopped` 拉起新容器进程;非 Docker 模式会先安排新进程延迟启动,再关闭当前进程。
|
||||
|
||||
## 项目结构
|
||||
|
||||
```text
|
||||
backend/ Rust/Axum 后端、SQLite migration、集成测试
|
||||
frontend/ Vue 3 管理端
|
||||
deploy/ Dockerfile、Compose、部署配置模板
|
||||
docs/ 架构、审计和维护文档
|
||||
release/ 版本、平台和发布脚本
|
||||
skills/ 项目本地 Codex skill
|
||||
```
|
||||
|
||||
## 开发检查
|
||||
|
||||
```bash
|
||||
cargo fmt --manifest-path backend/Cargo.toml --check
|
||||
cargo clippy --manifest-path backend/Cargo.toml --all-targets --all-features --locked -- -D warnings
|
||||
cargo test --manifest-path backend/Cargo.toml --test main --locked
|
||||
pnpm --dir frontend build
|
||||
docker compose -f deploy/docker-compose.yml config
|
||||
```
|
||||
模块边界、请求链路、存储归属和后台任务见 [架构说明](docs/architecture.md)。
|
||||
|
||||
## License
|
||||
|
||||
|
||||
Generated
+215
-500
File diff suppressed because it is too large
Load Diff
+22
-13
@@ -1,27 +1,33 @@
|
||||
[package]
|
||||
name = "codex-proxy-rs"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
rust-version = "1.95"
|
||||
build = "build/build.rs"
|
||||
rust-version = "1.97"
|
||||
|
||||
[dependencies]
|
||||
argon2 = "0.5.3"
|
||||
async-trait = "0.1.89"
|
||||
axum = "0.8.9"
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.11.1"
|
||||
chrono = { version = "0.4.45", features = ["serde", "clock"] }
|
||||
config = "0.15.23"
|
||||
dirs = "5.0.1"
|
||||
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
|
||||
dirs = "6.0.0"
|
||||
flate2 = "1.1.9"
|
||||
futures = "0.3.32"
|
||||
hex = "0.4.3"
|
||||
hmac = "0.13.0"
|
||||
hyper = { version = "1.10.1", features = ["http1", "server"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio"] }
|
||||
indexmap = { version = "2.0", features = ["serde"] }
|
||||
ipnet = "2.12.0"
|
||||
percent-encoding = "2.3.2"
|
||||
rand = "0.10.1"
|
||||
# Enables password_hash::rand_core::OsRng for argon2 0.5.
|
||||
rand_core = { version = "0.6.4", features = ["getrandom"] }
|
||||
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
|
||||
# Pinned to the Codex Desktop 26.707.41301 HTTP stack; re-audit a newer Desktop before upgrading.
|
||||
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "gzip", "brotli", "zstd", "deflate", "http2"] }
|
||||
rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] }
|
||||
rustls-native-certs = "0.8.4"
|
||||
@@ -30,12 +36,13 @@ secrecy = "0.10.3"
|
||||
semver = "1.0.27"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
sha2 = "0.10.9"
|
||||
sqlx = { version = "0.9.0", features = ["runtime-tokio", "tls-rustls", "sqlite", "chrono", "uuid"] }
|
||||
sha2 = "0.11.0"
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json"] }
|
||||
subtle = "2.6.1"
|
||||
tar = "0.4.44"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "signal", "fs", "time", "sync", "net", "test-util"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "signal", "fs", "time", "sync", "net"] }
|
||||
tokio-util = { version = "0.7.18", features = ["rt"] }
|
||||
tokio-tungstenite = { version = "0.28.0", features = ["proxy", "rustls-tls-native-roots"] }
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
tower-http = { version = "0.6.11", features = ["trace", "cors", "request-id", "timeout", "fs"] }
|
||||
@@ -43,19 +50,21 @@ tracing = "0.1.44"
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
|
||||
tungstenite = { version = "0.27.0", features = ["deflate", "proxy"] }
|
||||
url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v4", "v7", "serde"] }
|
||||
|
||||
[dev-dependencies]
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["derive"] }
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["test-util"] }
|
||||
wiremock = "0.6.5"
|
||||
|
||||
[patch.crates-io]
|
||||
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "132f5b39c862e3a970f731d709608b3e6276d5f6" }
|
||||
tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev = "9200079d3b54a1ff51072e24d81fd354f085156f" }
|
||||
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186" }
|
||||
tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev = "4fffad30fe373adbdcffab9545e9e9bf4f2fc19f" }
|
||||
|
||||
[lints.rust]
|
||||
future_incompatible = { level = "warn", priority = -1 }
|
||||
nonstandard_style = { level = "deny", priority = -1 }
|
||||
warnings = "deny"
|
||||
unsafe_code = "forbid"
|
||||
|
||||
[lints.clippy]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.95"
|
||||
channel = "1.97"
|
||||
components = ["clippy", "rustfmt"]
|
||||
profile = "minimal"
|
||||
components = ["rustfmt", "clippy"]
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
//! 管理端账号 HTTP 模块。
|
||||
|
||||
pub(crate) mod quota_view;
|
||||
pub mod routes;
|
||||
pub mod service;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,11 +0,0 @@
|
||||
use super::AdminAccountService;
|
||||
|
||||
impl AdminAccountService {
|
||||
pub(crate) async fn usage_cookie_header(&self, account_id: &str) -> Option<String> {
|
||||
self.cookies
|
||||
.cookie_header_for_request(account_id, "chatgpt.com", "/codex/usage")
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
//! 管理端账号业务服务。
|
||||
|
||||
mod contracts;
|
||||
mod cookies;
|
||||
mod exporting;
|
||||
mod importing;
|
||||
mod lifecycle;
|
||||
pub(crate) mod oauth;
|
||||
mod quota;
|
||||
mod testing;
|
||||
|
||||
use std::sync::Arc as StdArc;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::{
|
||||
upstream::accounts::{
|
||||
cookies::SqliteCookieStore,
|
||||
pool::RuntimeAccountPoolService,
|
||||
store::SqliteAccountStore,
|
||||
token_refresh::{RuntimeRefreshPolicy, TokenRefresher},
|
||||
},
|
||||
upstream::models::service::ModelService,
|
||||
upstream::transport::CodexBackendClient,
|
||||
};
|
||||
|
||||
pub use contracts::{
|
||||
AdminAccountError, AdminAccountHealthCheck, AdminAccountMetadata, AdminAccountRefreshOutcome,
|
||||
AdminAccountRefreshResult, AdminAccountUpdate, BatchDeleteAccounts, ExportedAccounts,
|
||||
ImportedAccounts, OAuthAuthorizeResult, OAuthExchangeInput,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AdminAccountService {
|
||||
pub store: SqliteAccountStore,
|
||||
pub(crate) cookies: SqliteCookieStore,
|
||||
pub(crate) codex: StdArc<CodexBackendClient>,
|
||||
pub(crate) models: StdArc<ModelService>,
|
||||
pub(crate) account_pool: StdArc<RuntimeAccountPoolService>,
|
||||
pub(crate) token_refresher: StdArc<dyn TokenRefresher>,
|
||||
pub(crate) oauth: oauth::AccountOAuthService,
|
||||
pub(crate) refresh_policy: RuntimeRefreshPolicy,
|
||||
pub(crate) installation_id: Option<String>,
|
||||
}
|
||||
|
||||
pub struct AdminAccountServiceParts {
|
||||
pub store: SqliteAccountStore,
|
||||
pub cookies: SqliteCookieStore,
|
||||
pub codex: StdArc<CodexBackendClient>,
|
||||
pub models: StdArc<ModelService>,
|
||||
pub account_pool: StdArc<RuntimeAccountPoolService>,
|
||||
pub token_refresher: StdArc<dyn TokenRefresher>,
|
||||
pub oauth: oauth::AccountOAuthService,
|
||||
pub refresh_policy: RuntimeRefreshPolicy,
|
||||
pub installation_id: Option<String>,
|
||||
}
|
||||
|
||||
impl AdminAccountService {
|
||||
pub fn new(parts: AdminAccountServiceParts) -> Self {
|
||||
Self {
|
||||
store: parts.store,
|
||||
cookies: parts.cookies,
|
||||
codex: parts.codex,
|
||||
models: parts.models,
|
||||
account_pool: parts.account_pool,
|
||||
token_refresher: parts.token_refresher,
|
||||
oauth: parts.oauth,
|
||||
refresh_policy: parts.refresh_policy,
|
||||
installation_id: parts.installation_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn next_refresh_at_for_expires_at(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expires_at: DateTime<chrono::Utc>,
|
||||
) -> DateTime<chrono::Utc> {
|
||||
crate::upstream::accounts::token_refresh::jittered_refresh_at(
|
||||
account_id,
|
||||
expires_at,
|
||||
self.refresh_policy.refresh_margin_seconds(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn refresh_tokens_from_refresh_token(
|
||||
&self,
|
||||
refresh_token: &str,
|
||||
) -> Result<contracts::RefreshedAccountTokens, AdminAccountError> {
|
||||
let token_pair = self
|
||||
.token_refresher
|
||||
.refresh(refresh_token)
|
||||
.await
|
||||
.map_err(AdminAccountError::RefreshTokenExchange)?;
|
||||
let access_token = crate::upstream::accounts::importing::normalize_nonempty(Some(
|
||||
crate::upstream::accounts::importing::normalize_bearer_token(&token_pair.access_token),
|
||||
))
|
||||
.ok_or(AdminAccountError::TokenRequired)?;
|
||||
let claims = crate::upstream::accounts::token_refresh::manual_account_claims(
|
||||
&access_token,
|
||||
Utc::now(),
|
||||
)
|
||||
.map_err(AdminAccountError::InvalidToken)?;
|
||||
|
||||
Ok(contracts::RefreshedAccountTokens {
|
||||
access_token,
|
||||
refresh_token: token_pair.refresh_token,
|
||||
claims,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn sync_account_pool(
|
||||
&self,
|
||||
account_id: &str,
|
||||
) -> Result<(), AdminAccountError> {
|
||||
self.account_pool
|
||||
.sync_account_from_repository(account_id)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|_| AdminAccountError::SyncAccountPool)
|
||||
}
|
||||
|
||||
pub(crate) async fn sync_account_pool_best_effort(&self, account_id: &str, operation: &str) {
|
||||
if let Err(error) = self
|
||||
.account_pool
|
||||
.sync_account_from_repository(account_id)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
operation,
|
||||
error = %error,
|
||||
"failed to sync runtime account pool after admin account update"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn evict_account_websocket_pool(&self, account_id: &str) {
|
||||
self.codex.evict_websocket_account(account_id).await;
|
||||
match self.store.get(account_id).await {
|
||||
Ok(Some(account)) => {
|
||||
if let Some(upstream_account_id) = account
|
||||
.account_id
|
||||
.as_deref()
|
||||
.filter(|value| *value != account_id)
|
||||
{
|
||||
self.codex
|
||||
.evict_websocket_account(upstream_account_id)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
error = %error,
|
||||
"failed to inspect account while evicting websocket pool"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
use super::{contracts::AdminAccountError, AdminAccountService};
|
||||
|
||||
impl AdminAccountService {
|
||||
pub async fn account_quota(
|
||||
&self,
|
||||
account_id: &str,
|
||||
) -> Result<serde_json::Value, AdminAccountError> {
|
||||
let stored = self
|
||||
.store
|
||||
.get(account_id)
|
||||
.await
|
||||
.map_err(|_| AdminAccountError::NotFound)?
|
||||
.ok_or(AdminAccountError::NotFound)?;
|
||||
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let token = stored.access_token.expose_secret().to_string();
|
||||
let cookie_header = self.usage_cookie_header(account_id).await;
|
||||
let context = crate::upstream::transport::CodexRequestContext {
|
||||
access_token: &token,
|
||||
account_id: stored.account_id.as_deref(),
|
||||
request_id: &request_id,
|
||||
turn_state: None,
|
||||
turn_metadata: None,
|
||||
beta_features: None,
|
||||
include_timing_metrics: None,
|
||||
version: None,
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
session_id: None,
|
||||
};
|
||||
let raw = self
|
||||
.codex
|
||||
.fetch_usage(context)
|
||||
.await
|
||||
.map_err(|e| AdminAccountError::FetchQuota(e.to_string()))?;
|
||||
let normalized = crate::upstream::accounts::quota::quota_from_usage(&raw);
|
||||
if let Ok(json_str) = serde_json::to_string(&normalized) {
|
||||
if matches!(
|
||||
self.store.update_quota_json(account_id, &json_str).await,
|
||||
Ok(true)
|
||||
) {
|
||||
self.sync_account_pool_best_effort(account_id, "account quota refresh")
|
||||
.await;
|
||||
}
|
||||
}
|
||||
Ok(serde_json::json!({ "quota": normalized, "raw": raw }))
|
||||
}
|
||||
}
|
||||
@@ -1,382 +0,0 @@
|
||||
use std::{convert::Infallible, pin::Pin};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::upstream::{
|
||||
accounts::{model::Account, store::StoredAccount},
|
||||
models::{info::CodexModelInfo, service::ModelRefreshPlanAccount},
|
||||
protocol::{
|
||||
responses::CodexResponsesRequest,
|
||||
sse::{encode_sse_event, parse_sse_events, SseEvent},
|
||||
},
|
||||
transport::CodexRequestContext,
|
||||
};
|
||||
|
||||
use super::{contracts::AdminAccountError, AdminAccountService};
|
||||
|
||||
pub(super) type AccountTestStream = Pin<Box<dyn Stream<Item = Result<Bytes, Infallible>> + Send>>;
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountModelOption {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
impl AdminAccountService {
|
||||
pub async fn account_models(
|
||||
&self,
|
||||
account_id: &str,
|
||||
) -> Result<Vec<AccountModelOption>, AdminAccountError> {
|
||||
let account = self
|
||||
.store
|
||||
.get(account_id)
|
||||
.await
|
||||
.map_err(|_| AdminAccountError::Inspect)?
|
||||
.ok_or(AdminAccountError::NotFound)?;
|
||||
let plan_type = account_plan_type(&account);
|
||||
let mut models = self.models.catalog().await.models_for_plan(&plan_type);
|
||||
if models.is_empty() {
|
||||
self.refresh_account_plan_models(&account, &plan_type).await;
|
||||
models = self.models.catalog().await.models_for_plan(&plan_type);
|
||||
}
|
||||
let models = models.iter().map(account_model_option).collect::<Vec<_>>();
|
||||
if models.is_empty() {
|
||||
return Err(AdminAccountError::NoModels);
|
||||
}
|
||||
Ok(models)
|
||||
}
|
||||
|
||||
async fn refresh_account_plan_models(&self, account: &StoredAccount, plan_type: &str) {
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let plan_account = ModelRefreshPlanAccount {
|
||||
plan_type: plan_type.to_string(),
|
||||
account: stored_account_to_model_refresh_account(account.clone()),
|
||||
};
|
||||
if let Err(error) = self
|
||||
.models
|
||||
.refresh_backend_models_with_installation_id(
|
||||
&[plan_account],
|
||||
&request_id,
|
||||
self.installation_id.as_deref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
plan_type,
|
||||
error = %error,
|
||||
"failed to refresh account plan models"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn test_connection_stream(
|
||||
&self,
|
||||
account_id: &str,
|
||||
model: String,
|
||||
) -> Result<AccountTestStream, AdminAccountError> {
|
||||
let account = self
|
||||
.store
|
||||
.get(account_id)
|
||||
.await
|
||||
.map_err(|_| AdminAccountError::Inspect)?
|
||||
.ok_or(AdminAccountError::NotFound)?;
|
||||
|
||||
let token = account.access_token.expose_secret().to_string();
|
||||
let upstream_account_id = account.account_id.clone();
|
||||
let cookie_header = self
|
||||
.cookies
|
||||
.cookie_header_for_request(&account.id, "chatgpt.com", "/codex/responses")
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let installation_id = self.installation_id.clone();
|
||||
let codex = self.codex.clone();
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let (tx, rx) = mpsc::channel::<Bytes>(16);
|
||||
|
||||
tokio::spawn(async move {
|
||||
send_test_event(
|
||||
&tx,
|
||||
json!({
|
||||
"type": "test_start",
|
||||
"model": model.clone(),
|
||||
"text": "正在连接 Codex Responses"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let request = test_responses_request(model);
|
||||
send_test_event(
|
||||
&tx,
|
||||
json!({
|
||||
"type": "request",
|
||||
"payload": serde_json::to_value(&request).unwrap_or_else(|_| json!({}))
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let context = CodexRequestContext {
|
||||
access_token: &token,
|
||||
account_id: upstream_account_id.as_deref(),
|
||||
request_id: &request_id,
|
||||
turn_state: None,
|
||||
turn_metadata: None,
|
||||
beta_features: None,
|
||||
include_timing_metrics: None,
|
||||
version: None,
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: installation_id.as_deref(),
|
||||
session_id: None,
|
||||
};
|
||||
|
||||
match codex.create_response_stream(&request, context).await {
|
||||
Ok(response) => process_upstream_test_stream(response.body, tx).await,
|
||||
Err(error) => {
|
||||
send_test_event(
|
||||
&tx,
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": error.to_string()
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let stream = futures::stream::unfold(rx, |mut rx| async {
|
||||
rx.recv()
|
||||
.await
|
||||
.map(|bytes| (Ok::<Bytes, Infallible>(bytes), rx))
|
||||
});
|
||||
Ok(Box::pin(stream))
|
||||
}
|
||||
}
|
||||
|
||||
fn account_model_option(model: &CodexModelInfo) -> AccountModelOption {
|
||||
AccountModelOption {
|
||||
id: model.id.clone(),
|
||||
label: model.display_name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn account_plan_type(account: &StoredAccount) -> String {
|
||||
account
|
||||
.plan_type
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or("default")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn stored_account_to_model_refresh_account(stored: StoredAccount) -> Account {
|
||||
Account {
|
||||
id: stored.id,
|
||||
email: stored.email,
|
||||
account_id: stored.account_id,
|
||||
user_id: stored.user_id,
|
||||
label: stored.label,
|
||||
plan_type: stored.plan_type,
|
||||
access_token: stored.access_token.expose_secret().to_string(),
|
||||
refresh_token: stored
|
||||
.refresh_token
|
||||
.map(|token| token.expose_secret().to_string()),
|
||||
access_token_expires_at: stored.access_token_expires_at,
|
||||
next_refresh_at: stored.next_refresh_at,
|
||||
status: stored.status,
|
||||
quota_limit_reached: false,
|
||||
quota_verify_required: false,
|
||||
quota_cooldown_until: None,
|
||||
cloudflare_cooldown_until: None,
|
||||
request_count: 0,
|
||||
empty_response_count: 0,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
window_request_count: 0,
|
||||
window_input_tokens: 0,
|
||||
window_output_tokens: 0,
|
||||
window_cached_tokens: 0,
|
||||
window_image_input_tokens: 0,
|
||||
window_image_output_tokens: 0,
|
||||
window_image_request_count: 0,
|
||||
window_image_request_failed_count: 0,
|
||||
window_started_at: None,
|
||||
window_reset_at: None,
|
||||
limit_window_seconds: None,
|
||||
added_at: stored.added_at,
|
||||
last_used_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_responses_request(model: String) -> CodexResponsesRequest {
|
||||
let mut request = CodexResponsesRequest::new_http_sse(
|
||||
model,
|
||||
"You are checking whether this Codex account can answer. Reply with ok.",
|
||||
vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "hi"
|
||||
}
|
||||
]
|
||||
})],
|
||||
);
|
||||
request.force_http_sse = true;
|
||||
request
|
||||
}
|
||||
|
||||
async fn process_upstream_test_stream(
|
||||
mut body: crate::upstream::transport::CodexBackendSseStream,
|
||||
tx: mpsc::Sender<Bytes>,
|
||||
) {
|
||||
let mut buffer = String::new();
|
||||
|
||||
while let Some(chunk) = body.next().await {
|
||||
let chunk = match chunk {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
send_test_event(&tx, json!({ "type": "error", "error": error.to_string() })).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
buffer.push_str(&String::from_utf8_lossy(&chunk));
|
||||
|
||||
while let Some(frame) = take_sse_frame(&mut buffer) {
|
||||
if process_sse_frame(&tx, &frame).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !buffer.trim().is_empty() && process_sse_frame(&tx, &buffer).await {
|
||||
return;
|
||||
}
|
||||
|
||||
send_test_event(
|
||||
&tx,
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "Stream ended before response.completed"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn process_sse_frame(tx: &mpsc::Sender<Bytes>, frame: &str) -> bool {
|
||||
let events = match parse_sse_events(frame) {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
send_test_event(
|
||||
tx,
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": error.to_string()
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
for event in events {
|
||||
if process_sse_event(tx, &event).await {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn process_sse_event(tx: &mpsc::Sender<Bytes>, event: &SseEvent) -> bool {
|
||||
let value: Value = match serde_json::from_str(&event.data) {
|
||||
Ok(value) => value,
|
||||
Err(_) => return false,
|
||||
};
|
||||
match value.get("type").and_then(Value::as_str) {
|
||||
Some("response.output_text.delta") => {
|
||||
if let Some(delta) = value.get("delta").and_then(Value::as_str) {
|
||||
if !delta.is_empty() {
|
||||
send_test_event(tx, json!({ "type": "content", "text": delta })).await;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
Some("response.completed" | "response.done") => {
|
||||
send_test_event(tx, json!({ "type": "test_complete", "success": true })).await;
|
||||
true
|
||||
}
|
||||
Some("response.failed") => {
|
||||
send_test_event(
|
||||
tx,
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": response_failure_message(&value)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
true
|
||||
}
|
||||
Some("error") => {
|
||||
send_test_event(
|
||||
tx,
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": error_event_message(&value)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn response_failure_message(value: &Value) -> String {
|
||||
value
|
||||
.pointer("/response/error/message")
|
||||
.or_else(|| value.pointer("/error/message"))
|
||||
.or_else(|| value.get("message"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|message| !message.trim().is_empty())
|
||||
.unwrap_or("OpenAI response failed")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn error_event_message(value: &Value) -> String {
|
||||
value
|
||||
.pointer("/error/message")
|
||||
.or_else(|| value.get("message"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|message| !message.trim().is_empty())
|
||||
.unwrap_or("Unknown upstream error")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn take_sse_frame(buffer: &mut String) -> Option<String> {
|
||||
let index = buffer.find("\n\n").or_else(|| buffer.find("\r\n\r\n"))?;
|
||||
let delimiter_len = if buffer[index..].starts_with("\r\n\r\n") {
|
||||
4
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let frame = buffer[..index + delimiter_len].to_string();
|
||||
buffer.drain(..index + delimiter_len);
|
||||
Some(frame)
|
||||
}
|
||||
|
||||
async fn send_test_event(tx: &mpsc::Sender<Bytes>, event: Value) {
|
||||
let _ = tx
|
||||
.send(Bytes::from(encode_sse_event("", &event.to_string())))
|
||||
.await;
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
//! 管理端认证与本地会话模块。
|
||||
|
||||
pub mod service;
|
||||
pub mod session;
|
||||
@@ -1,341 +0,0 @@
|
||||
//! 管理员会话存储与认证领域逻辑。
|
||||
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use sqlx::SqlitePool;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::infra::identity::{hash_admin_password, verify_admin_password};
|
||||
|
||||
const LOGIN_FAILURE_WINDOW: Duration = Duration::minutes(15);
|
||||
const LOGIN_LOCK_DURATION: Duration = Duration::minutes(15);
|
||||
const LOGIN_MAX_FAILURES: u32 = 5;
|
||||
|
||||
/// 管理员登录成功后的会话。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AdminLoginSession {
|
||||
/// 会话 ID。
|
||||
pub session_id: String,
|
||||
/// 过期时间。
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 管理员会话错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AdminSessionError {
|
||||
/// 校验管理员会话失败。
|
||||
#[error("failed to validate admin session")]
|
||||
Validate,
|
||||
/// 计算管理员密码哈希失败。
|
||||
#[error("failed to hash admin password")]
|
||||
HashPassword,
|
||||
/// 创建默认管理员失败。
|
||||
#[error("failed to create default admin user")]
|
||||
CreateAdmin,
|
||||
/// 读取管理员用户失败。
|
||||
#[error("failed to load admin user")]
|
||||
LoadAdmin,
|
||||
/// 校验管理员密码失败。
|
||||
#[error("failed to verify admin password")]
|
||||
VerifyPassword,
|
||||
/// 创建管理员会话失败。
|
||||
#[error("failed to create admin session")]
|
||||
CreateSession,
|
||||
/// 删除管理员会话失败。
|
||||
#[error("failed to delete admin session")]
|
||||
DeleteSession,
|
||||
/// 管理员登录尝试过于频繁。
|
||||
#[error("too many admin login attempts")]
|
||||
LoginThrottled,
|
||||
}
|
||||
|
||||
/// 管理员会话服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AdminSessionService {
|
||||
store: SqliteAdminSessionStore,
|
||||
default_username: String,
|
||||
session_ttl_minutes: u64,
|
||||
login_failures: Arc<Mutex<HashMap<String, LoginFailureState>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct LoginFailureState {
|
||||
failures: u32,
|
||||
first_failed_at: DateTime<Utc>,
|
||||
locked_until: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl AdminSessionService {
|
||||
/// 构造服务。
|
||||
pub fn new(
|
||||
store: SqliteAdminSessionStore,
|
||||
default_username: String,
|
||||
session_ttl_minutes: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
default_username,
|
||||
session_ttl_minutes,
|
||||
login_failures: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验管理员会话。
|
||||
pub async fn validate(&self, session_id: Option<&str>) -> Result<bool, AdminSessionError> {
|
||||
let Some(session_id) = session_id else {
|
||||
return Ok(false);
|
||||
};
|
||||
self.store
|
||||
.validate_session(session_id)
|
||||
.await
|
||||
.map_err(|_| AdminSessionError::Validate)
|
||||
}
|
||||
|
||||
/// 确保默认管理员存在。
|
||||
pub async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminSessionError> {
|
||||
let password_hash =
|
||||
hash_admin_password(password).map_err(|_| AdminSessionError::HashPassword)?;
|
||||
self.store
|
||||
.ensure_default_admin(&password_hash)
|
||||
.await
|
||||
.map_err(|_| AdminSessionError::CreateAdmin)
|
||||
}
|
||||
|
||||
/// 登录并创建管理员会话。
|
||||
pub async fn login(
|
||||
&self,
|
||||
source: &str,
|
||||
username: Option<&str>,
|
||||
password: &str,
|
||||
) -> Result<Option<AdminLoginSession>, AdminSessionError> {
|
||||
let source = login_source_key(source);
|
||||
self.ensure_login_allowed(&source).await?;
|
||||
|
||||
let username = username.unwrap_or(&self.default_username);
|
||||
if username != self.default_username.as_str() {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(admin) = self
|
||||
.store
|
||||
.load_first_admin()
|
||||
.await
|
||||
.map_err(|_| AdminSessionError::LoadAdmin)?
|
||||
else {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
};
|
||||
let password_matches = verify_admin_password(password, &admin.password_hash)
|
||||
.map_err(|_| AdminSessionError::VerifyPassword)?;
|
||||
if !password_matches {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
}
|
||||
self.clear_failed_login(&source).await;
|
||||
let session_id = format!("sess_{}", Uuid::new_v4().simple());
|
||||
let ttl_minutes = self.session_ttl_minutes.min(i64::MAX as u64) as i64;
|
||||
let expires_at = Utc::now() + Duration::minutes(ttl_minutes);
|
||||
self.store
|
||||
.create_session(&session_id, &admin.id, expires_at)
|
||||
.await
|
||||
.map_err(|_| AdminSessionError::CreateSession)?;
|
||||
Ok(Some(AdminLoginSession {
|
||||
session_id,
|
||||
expires_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 删除管理员会话。
|
||||
pub async fn delete_session(&self, session_id: &str) -> Result<bool, AdminSessionError> {
|
||||
self.store
|
||||
.delete_session(session_id)
|
||||
.await
|
||||
.map_err(|_| AdminSessionError::DeleteSession)
|
||||
}
|
||||
|
||||
async fn ensure_login_allowed(&self, source: &str) -> Result<(), AdminSessionError> {
|
||||
let now = Utc::now();
|
||||
let mut failures = self.login_failures.lock().await;
|
||||
retain_active_login_failures(&mut failures, now);
|
||||
if failures
|
||||
.get(source)
|
||||
.and_then(|state| state.locked_until)
|
||||
.is_some_and(|locked_until| locked_until > now)
|
||||
{
|
||||
return Err(AdminSessionError::LoginThrottled);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_failed_login(&self, source: &str) {
|
||||
let now = Utc::now();
|
||||
let mut failures = self.login_failures.lock().await;
|
||||
retain_active_login_failures(&mut failures, now);
|
||||
let state = failures
|
||||
.entry(source.to_string())
|
||||
.or_insert_with(|| LoginFailureState {
|
||||
failures: 0,
|
||||
first_failed_at: now,
|
||||
locked_until: None,
|
||||
});
|
||||
if now - state.first_failed_at > LOGIN_FAILURE_WINDOW {
|
||||
state.failures = 0;
|
||||
state.first_failed_at = now;
|
||||
state.locked_until = None;
|
||||
}
|
||||
state.failures += 1;
|
||||
if state.failures >= LOGIN_MAX_FAILURES {
|
||||
state.locked_until = Some(now + LOGIN_LOCK_DURATION);
|
||||
}
|
||||
}
|
||||
|
||||
async fn clear_failed_login(&self, source: &str) {
|
||||
self.login_failures.lock().await.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
fn login_source_key(source: &str) -> String {
|
||||
let source = source.trim();
|
||||
if source.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
source.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn retain_active_login_failures(
|
||||
failures: &mut HashMap<String, LoginFailureState>,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
failures.retain(|_, state| {
|
||||
state
|
||||
.locked_until
|
||||
.is_some_and(|locked_until| locked_until > now)
|
||||
|| now - state.first_failed_at <= LOGIN_FAILURE_WINDOW
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 已持久化的管理用户
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 已持久化的管理员用户。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StoredAdminUser {
|
||||
/// 管理员用户 ID。
|
||||
pub id: String,
|
||||
/// 管理员密码哈希。
|
||||
pub password_hash: String,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQLite 管理员会话存储
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// SQLite 管理员会话存储结果。
|
||||
pub type SqliteAdminSessionStoreResult<T> = Result<T, sqlx::Error>;
|
||||
|
||||
/// SQLite 管理员会话存储。
|
||||
#[derive(Clone)]
|
||||
pub struct SqliteAdminSessionStore {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteAdminSessionStore {
|
||||
/// 构造存储。
|
||||
pub fn new(pool: SqlitePool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// 如果还没有管理员用户,则创建默认管理员。
|
||||
pub async fn ensure_default_admin(
|
||||
&self,
|
||||
password_hash: &str,
|
||||
) -> SqliteAdminSessionStoreResult<bool> {
|
||||
let existing: (i64,) = sqlx::query_as("select count(*) from admin_users")
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
if existing.0 > 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let now = Utc::now().to_rfc3339();
|
||||
sqlx::query(
|
||||
"insert into admin_users (id, password_hash, created_at, updated_at) values (?, ?, ?, ?)",
|
||||
)
|
||||
.bind("admin_1")
|
||||
.bind(password_hash)
|
||||
.bind(&now)
|
||||
.bind(&now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// 读取第一个管理员用户。
|
||||
pub async fn load_first_admin(&self) -> SqliteAdminSessionStoreResult<Option<StoredAdminUser>> {
|
||||
let row = sqlx::query_as::<_, (String, String)>(
|
||||
"select id, password_hash from admin_users order by created_at asc, id asc limit 1",
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(row.map(|(id, password_hash)| StoredAdminUser { id, password_hash }))
|
||||
}
|
||||
|
||||
/// 创建管理员会话。
|
||||
pub async fn create_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
user_id: &str,
|
||||
expires_at: DateTime<Utc>,
|
||||
) -> SqliteAdminSessionStoreResult<()> {
|
||||
let now = Utc::now().to_rfc3339();
|
||||
sqlx::query(
|
||||
"insert into admin_sessions (id, user_id, expires_at, created_at) values (?, ?, ?, ?)",
|
||||
)
|
||||
.bind(session_id)
|
||||
.bind(user_id)
|
||||
.bind(expires_at.to_rfc3339())
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验管理员会话是否存在且未过期。
|
||||
pub async fn validate_session(&self, session_id: &str) -> SqliteAdminSessionStoreResult<bool> {
|
||||
let now = Utc::now().to_rfc3339();
|
||||
let row = sqlx::query("select 1 from admin_sessions where id = ? and expires_at > ?")
|
||||
.bind(session_id)
|
||||
.bind(now)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
Ok(row.is_some())
|
||||
}
|
||||
|
||||
/// 删除已经过期的管理员会话。
|
||||
pub async fn cleanup_expired_sessions(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> SqliteAdminSessionStoreResult<u64> {
|
||||
let result = sqlx::query("delete from admin_sessions where expires_at <= ?")
|
||||
.bind(now.to_rfc3339())
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
/// 删除指定的管理员会话。
|
||||
pub async fn delete_session(&self, session_id: &str) -> SqliteAdminSessionStoreResult<bool> {
|
||||
let result = sqlx::query("delete from admin_sessions where id = ?")
|
||||
.bind(session_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
//! 管理端 v1 接口访问 Key 模块。
|
||||
|
||||
pub mod routes;
|
||||
pub mod service;
|
||||
@@ -1,698 +0,0 @@
|
||||
//! 客户端 API Key 端口定义、业务服务与 SQLite 存储适配器。
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, Mutex, RwLock,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use sqlx::{Row, SqlitePool};
|
||||
use thiserror::Error;
|
||||
use tokio::time::sleep;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::infra::{
|
||||
identity::generate_client_api_key,
|
||||
json::{decode_cursor, encode_cursor, Page},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 端口定义
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 客户端 API Key 存储错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ClientKeyStoreError {
|
||||
/// 底层存储失败。
|
||||
#[error("client key store operation failed: {message}")]
|
||||
OperationFailed {
|
||||
/// 错误说明。
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// 客户端 API Key 存储结果类型。
|
||||
pub type ClientKeyStoreResult<T> = Result<T, ClientKeyStoreError>;
|
||||
|
||||
/// 提供客户端 API Key 验证能力的端口。
|
||||
#[async_trait]
|
||||
pub trait ClientKeyStore: Send + Sync + 'static {
|
||||
/// 验证明文客户端 API Key,并在成功时记录使用时间。
|
||||
async fn verify_and_touch(&self, plaintext: &str) -> ClientKeyStoreResult<bool>;
|
||||
}
|
||||
|
||||
const CLIENT_KEY_LAST_USED_FLUSH_DELAY: Duration = Duration::from_secs(1);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 业务服务
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 客户端 API Key 服务。
|
||||
#[derive(Clone)]
|
||||
pub struct ClientKeyService {
|
||||
store: Arc<RuntimeClientKeyStore>,
|
||||
}
|
||||
|
||||
impl ClientKeyService {
|
||||
/// 构造服务。
|
||||
pub fn new(store: Arc<RuntimeClientKeyStore>) -> Self {
|
||||
Self { store }
|
||||
}
|
||||
|
||||
/// 验证客户端 API Key。
|
||||
pub async fn verify(&self, plaintext: &str) -> ClientKeyStoreResult<bool> {
|
||||
if !plaintext.starts_with("sk_") {
|
||||
return Ok(false);
|
||||
}
|
||||
self.store.verify_and_touch(plaintext).await
|
||||
}
|
||||
|
||||
/// 重新加载运行时内存鉴权表。
|
||||
pub async fn reload_from_store(&self) -> ClientKeyStoreResult<()> {
|
||||
self.store.reload_from_store().await
|
||||
}
|
||||
|
||||
/// 立即刷写待持久化的最近使用时间。
|
||||
pub async fn flush_pending_last_used(&self) {
|
||||
self.store.flush_pending_last_used().await;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 管理端业务服务
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 管理端客户端 API Key 服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AdminClientKeyService {
|
||||
store: SqliteClientKeyStore,
|
||||
runtime: Arc<RuntimeClientKeyStore>,
|
||||
}
|
||||
|
||||
impl AdminClientKeyService {
|
||||
/// 构造服务。
|
||||
pub fn new(store: SqliteClientKeyStore, runtime: Arc<RuntimeClientKeyStore>) -> Self {
|
||||
Self { store, runtime }
|
||||
}
|
||||
|
||||
/// 创建客户端 API Key。
|
||||
pub async fn create(&self, name: &str) -> Result<AdminClientApiKey, AdminClientKeyError> {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return Err(AdminClientKeyError::EmptyName);
|
||||
}
|
||||
self.store
|
||||
.create(name)
|
||||
.await
|
||||
.inspect(|created| self.runtime.upsert_stored(created))
|
||||
.map(AdminClientApiKey::from)
|
||||
.map_err(|_| AdminClientKeyError::Create)
|
||||
}
|
||||
|
||||
/// 分页列出客户端 API Key。
|
||||
pub async fn list(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: u32,
|
||||
) -> Result<Page<AdminClientApiKey>, AdminClientKeyError> {
|
||||
let page = self
|
||||
.store
|
||||
.list(cursor, limit)
|
||||
.await
|
||||
.map_err(|_| AdminClientKeyError::List)?;
|
||||
Ok(Page {
|
||||
items: page
|
||||
.items
|
||||
.into_iter()
|
||||
.map(AdminClientApiKey::from)
|
||||
.collect(),
|
||||
next_cursor: page.next_cursor,
|
||||
})
|
||||
}
|
||||
|
||||
/// 按 ID 读取客户端 API Key 元数据。
|
||||
pub async fn get(
|
||||
&self,
|
||||
key_id: &str,
|
||||
) -> Result<Option<AdminClientApiKey>, AdminClientKeyError> {
|
||||
self.store
|
||||
.get(key_id)
|
||||
.await
|
||||
.map(|key| key.map(AdminClientApiKey::from))
|
||||
.map_err(|_| AdminClientKeyError::List)
|
||||
}
|
||||
|
||||
/// 更新客户端 API Key 标签。
|
||||
pub async fn update_label(
|
||||
&self,
|
||||
key_id: &str,
|
||||
label: Option<String>,
|
||||
) -> Result<Option<AdminClientApiKey>, AdminClientKeyError> {
|
||||
if label.as_ref().is_some_and(|l| l.chars().count() > 64) {
|
||||
return Err(AdminClientKeyError::LabelTooLong);
|
||||
}
|
||||
self.store
|
||||
.set_label(key_id, label)
|
||||
.await
|
||||
.map(|key| key.map(AdminClientApiKey::from))
|
||||
.map_err(|_| AdminClientKeyError::UpdateLabel)
|
||||
}
|
||||
|
||||
/// 更新客户端 API Key 启用状态。
|
||||
pub async fn update_status(
|
||||
&self,
|
||||
key_id: &str,
|
||||
status: &str,
|
||||
) -> Result<bool, AdminClientKeyError> {
|
||||
let enabled = parse_client_key_status(status)?;
|
||||
let updated = self
|
||||
.store
|
||||
.set_enabled(key_id, enabled)
|
||||
.await
|
||||
.map_err(|_| AdminClientKeyError::UpdateStatus)?;
|
||||
if updated {
|
||||
if enabled {
|
||||
let Some(key) = self
|
||||
.store
|
||||
.get(key_id)
|
||||
.await
|
||||
.map_err(|_| AdminClientKeyError::UpdateStatus)?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
self.runtime.upsert_stored(&key);
|
||||
} else {
|
||||
self.runtime.remove_by_id(key_id);
|
||||
}
|
||||
}
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
/// 批量删除客户端 API Key。
|
||||
pub async fn batch_delete(
|
||||
&self,
|
||||
ids: Vec<String>,
|
||||
) -> Result<BatchDeleteClientApiKeys, AdminClientKeyError> {
|
||||
if ids.is_empty() {
|
||||
return Err(AdminClientKeyError::EmptyIds);
|
||||
}
|
||||
let mut deleted = 0u32;
|
||||
let mut not_found = Vec::new();
|
||||
for id in ids {
|
||||
match self.store.delete(&id).await {
|
||||
Ok(true) => {
|
||||
self.runtime.remove_by_id(&id);
|
||||
deleted += 1;
|
||||
}
|
||||
Ok(false) => not_found.push(id),
|
||||
Err(_) => return Err(AdminClientKeyError::Delete),
|
||||
}
|
||||
}
|
||||
Ok(BatchDeleteClientApiKeys { deleted, not_found })
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端可见的客户端 API Key。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AdminClientApiKey {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub prefix: String,
|
||||
pub key: String,
|
||||
pub enabled: bool,
|
||||
pub created_at: String,
|
||||
pub last_used_at: Option<String>,
|
||||
}
|
||||
|
||||
/// 客户端 API Key 批量删除结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BatchDeleteClientApiKeys {
|
||||
pub deleted: u32,
|
||||
pub not_found: Vec<String>,
|
||||
}
|
||||
|
||||
/// 管理端客户端 API Key 错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AdminClientKeyError {
|
||||
#[error("failed to list client API keys")]
|
||||
List,
|
||||
#[error("failed to create client API key")]
|
||||
Create,
|
||||
#[error("failed to delete client API key")]
|
||||
Delete,
|
||||
#[error("failed to update client API key label")]
|
||||
UpdateLabel,
|
||||
#[error("failed to update client API key status")]
|
||||
UpdateStatus,
|
||||
#[error("unsupported client API key status: {0}")]
|
||||
InvalidStatus(String),
|
||||
#[error("client API key name is required")]
|
||||
EmptyName,
|
||||
#[error("client API key ids are required")]
|
||||
EmptyIds,
|
||||
#[error("client API key label must be 64 characters or fewer")]
|
||||
LabelTooLong,
|
||||
}
|
||||
|
||||
fn parse_client_key_status(status: &str) -> Result<bool, AdminClientKeyError> {
|
||||
match status.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Ok(true),
|
||||
"disabled" => Ok(false),
|
||||
other => Err(AdminClientKeyError::InvalidStatus(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredClientApiKey> for AdminClientApiKey {
|
||||
fn from(key: StoredClientApiKey) -> Self {
|
||||
Self {
|
||||
id: key.id,
|
||||
name: key.name,
|
||||
label: key.label,
|
||||
prefix: key.prefix,
|
||||
key: key.key,
|
||||
enabled: key.enabled,
|
||||
created_at: key.created_at,
|
||||
last_used_at: key.last_used_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQLite 存储适配器
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// SQLite 客户端 API Key 存储错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SqliteClientKeyStoreError {
|
||||
/// 数据库错误。
|
||||
#[error("sqlite client key database error: {0}")]
|
||||
Database(#[from] sqlx::Error),
|
||||
/// 分页游标无效。
|
||||
#[error("invalid client key pagination cursor")]
|
||||
InvalidCursor,
|
||||
}
|
||||
|
||||
/// 已持久化的客户端 API Key 元数据。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StoredClientApiKey {
|
||||
/// API Key 记录 ID。
|
||||
pub id: String,
|
||||
/// API Key 名称。
|
||||
pub name: String,
|
||||
/// 管理员可见标签。
|
||||
pub label: Option<String>,
|
||||
/// 明文 API Key 的短前缀。
|
||||
pub prefix: String,
|
||||
/// 管理端可复制的完整 API Key。
|
||||
pub key: String,
|
||||
/// 是否允许用于 `/v1` 认证。
|
||||
pub enabled: bool,
|
||||
/// 创建时间。
|
||||
pub created_at: String,
|
||||
/// 最近一次成功使用时间。
|
||||
pub last_used_at: Option<String>,
|
||||
}
|
||||
|
||||
/// SQLite 客户端 API Key 存储。
|
||||
#[derive(Clone)]
|
||||
pub struct SqliteClientKeyStore {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteClientKeyStore {
|
||||
/// 构造存储适配器。
|
||||
pub fn new(pool: SqlitePool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// 暴露底层连接池,供集成测试和运行时组合层复用。
|
||||
pub fn pool(&self) -> &SqlitePool {
|
||||
&self.pool
|
||||
}
|
||||
|
||||
/// 读取所有已启用客户端 API Key,用于运行时内存鉴权表初始化。
|
||||
pub async fn list_enabled(&self) -> Result<Vec<StoredClientApiKey>, SqliteClientKeyStoreError> {
|
||||
let rows = sqlx::query(
|
||||
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where enabled = 1 order by created_at asc, id asc",
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|row| Self::key_from_row(&row))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// 创建新的本地客户端 API Key。
|
||||
pub async fn create(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<StoredClientApiKey, SqliteClientKeyStoreError> {
|
||||
let generated = generate_client_api_key();
|
||||
let id = format!("key_{}", Uuid::new_v4().simple());
|
||||
let now = Utc::now().to_rfc3339();
|
||||
sqlx::query(
|
||||
"insert into client_api_keys (id, name, label, prefix, key, enabled, created_at, last_used_at) values (?, ?, null, ?, ?, 1, ?, null)",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(name)
|
||||
.bind(&generated.prefix)
|
||||
.bind(&generated.key)
|
||||
.bind(&now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(StoredClientApiKey {
|
||||
id,
|
||||
name: name.to_string(),
|
||||
label: None,
|
||||
prefix: generated.prefix,
|
||||
key: generated.key,
|
||||
enabled: true,
|
||||
created_at: now,
|
||||
last_used_at: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// 按创建时间倒序分页列出客户端 API Key。
|
||||
pub async fn list(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: u32,
|
||||
) -> Result<Page<StoredClientApiKey>, SqliteClientKeyStoreError> {
|
||||
let fetch_limit = i64::from(limit) + 1;
|
||||
let rows = if let Some(cursor) = cursor {
|
||||
let (created_at, id) =
|
||||
decode_cursor(&cursor).ok_or(SqliteClientKeyStoreError::InvalidCursor)?;
|
||||
sqlx::query(
|
||||
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where created_at < ? or (created_at = ? and id < ?) order by created_at desc, id desc limit ?",
|
||||
)
|
||||
.bind(&created_at)
|
||||
.bind(created_at)
|
||||
.bind(id)
|
||||
.bind(fetch_limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?
|
||||
} else {
|
||||
sqlx::query(
|
||||
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys order by created_at desc, id desc limit ?",
|
||||
)
|
||||
.bind(fetch_limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?
|
||||
};
|
||||
|
||||
let has_next = rows.len() > limit as usize;
|
||||
let take_count = rows.len().min(limit as usize);
|
||||
let items = rows
|
||||
.into_iter()
|
||||
.take(take_count)
|
||||
.map(|row| Self::key_from_row(&row))
|
||||
.collect::<Vec<_>>();
|
||||
let next_cursor = if has_next {
|
||||
items
|
||||
.last()
|
||||
.map(|key| encode_cursor(&key.created_at, &key.id))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Page { items, next_cursor })
|
||||
}
|
||||
|
||||
/// 按 ID 读取客户端 API Key 元数据。
|
||||
pub async fn get(
|
||||
&self,
|
||||
id: &str,
|
||||
) -> Result<Option<StoredClientApiKey>, SqliteClientKeyStoreError> {
|
||||
let row = sqlx::query(
|
||||
"select id, name, label, prefix, key, enabled, created_at, last_used_at from client_api_keys where id = ?",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(row.map(|row| Self::key_from_row(&row)))
|
||||
}
|
||||
|
||||
/// 更新客户端 API Key 启用状态。
|
||||
pub async fn set_enabled(
|
||||
&self,
|
||||
id: &str,
|
||||
enabled: bool,
|
||||
) -> Result<bool, SqliteClientKeyStoreError> {
|
||||
let result = sqlx::query("update client_api_keys set enabled = ? where id = ?")
|
||||
.bind(if enabled { 1_i64 } else { 0_i64 })
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
/// 更新客户端 API Key 标签并返回最新元数据。
|
||||
pub async fn set_label(
|
||||
&self,
|
||||
id: &str,
|
||||
label: Option<String>,
|
||||
) -> Result<Option<StoredClientApiKey>, SqliteClientKeyStoreError> {
|
||||
let result = sqlx::query("update client_api_keys set label = ? where id = ?")
|
||||
.bind(label)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
if result.rows_affected() == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
self.get(id).await
|
||||
}
|
||||
|
||||
/// 删除客户端 API Key。
|
||||
pub async fn delete(&self, id: &str) -> Result<bool, SqliteClientKeyStoreError> {
|
||||
let result = sqlx::query("delete from client_api_keys where id = ?")
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
/// 更新最近使用时间。
|
||||
pub async fn update_last_used_at(
|
||||
&self,
|
||||
id: &str,
|
||||
last_used_at: &str,
|
||||
) -> Result<bool, SqliteClientKeyStoreError> {
|
||||
let result = sqlx::query("update client_api_keys set last_used_at = ? where id = ?")
|
||||
.bind(last_used_at)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
fn key_from_row(row: &sqlx::sqlite::SqliteRow) -> StoredClientApiKey {
|
||||
StoredClientApiKey {
|
||||
id: row.get("id"),
|
||||
name: row.get("name"),
|
||||
label: row.get("label"),
|
||||
prefix: row.get("prefix"),
|
||||
key: row.get("key"),
|
||||
enabled: row.get::<i64, _>("enabled") != 0,
|
||||
created_at: row.get("created_at"),
|
||||
last_used_at: row.get("last_used_at"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ClientKeyStore for RuntimeClientKeyStore {
|
||||
async fn verify_and_touch(&self, plaintext: &str) -> ClientKeyStoreResult<bool> {
|
||||
let Some(id) = self.lookup_key_id(plaintext) else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
self.queue_last_used_touch(id);
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
/// 运行时客户端 API Key 内存鉴权表。
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeClientKeyStore {
|
||||
store: SqliteClientKeyStore,
|
||||
cache: Arc<RwLock<RuntimeClientKeyCache>>,
|
||||
pending_last_used: Arc<Mutex<BTreeMap<String, String>>>,
|
||||
flush_scheduled: Arc<AtomicBool>,
|
||||
flush_delay: Duration,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RuntimeClientKeyCache {
|
||||
keys_by_plaintext: HashMap<String, String>,
|
||||
plaintext_by_id: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl RuntimeClientKeyStore {
|
||||
/// 构造运行时内存鉴权表。
|
||||
pub fn new(store: SqliteClientKeyStore) -> Self {
|
||||
Self {
|
||||
store,
|
||||
cache: Arc::new(RwLock::new(RuntimeClientKeyCache::default())),
|
||||
pending_last_used: Arc::new(Mutex::new(BTreeMap::new())),
|
||||
flush_scheduled: Arc::new(AtomicBool::new(false)),
|
||||
flush_delay: CLIENT_KEY_LAST_USED_FLUSH_DELAY,
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 SQLite 加载当前启用的客户端 API Key。
|
||||
pub async fn reload_from_store(&self) -> ClientKeyStoreResult<()> {
|
||||
let keys = self
|
||||
.store
|
||||
.list_enabled()
|
||||
.await
|
||||
.map_err(|error| map_client_key_store_error(&error))?;
|
||||
let mut keys_by_plaintext = HashMap::with_capacity(keys.len());
|
||||
let mut plaintext_by_id = HashMap::with_capacity(keys.len());
|
||||
for key in keys {
|
||||
keys_by_plaintext.insert(key.key.clone(), key.id.clone());
|
||||
plaintext_by_id.insert(key.id, key.key);
|
||||
}
|
||||
|
||||
*self
|
||||
.cache
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = RuntimeClientKeyCache {
|
||||
keys_by_plaintext,
|
||||
plaintext_by_id,
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn lookup_key_id(&self, plaintext: &str) -> Option<String> {
|
||||
self.cache
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.keys_by_plaintext
|
||||
.get(plaintext)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn upsert_stored(&self, key: &StoredClientApiKey) {
|
||||
if key.enabled {
|
||||
self.upsert_enabled_key(&key.id, &key.key);
|
||||
} else {
|
||||
self.remove_by_id(&key.id);
|
||||
}
|
||||
}
|
||||
|
||||
fn upsert_enabled_key(&self, id: &str, plaintext: &str) {
|
||||
let mut cache = self
|
||||
.cache
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(old_plaintext) = cache.plaintext_by_id.remove(id) {
|
||||
cache.keys_by_plaintext.remove(&old_plaintext);
|
||||
}
|
||||
cache
|
||||
.keys_by_plaintext
|
||||
.insert(plaintext.to_string(), id.to_string());
|
||||
cache
|
||||
.plaintext_by_id
|
||||
.insert(id.to_string(), plaintext.to_string());
|
||||
}
|
||||
|
||||
fn remove_by_id(&self, id: &str) {
|
||||
let mut cache = self
|
||||
.cache
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(plaintext) = cache.plaintext_by_id.remove(id) {
|
||||
cache.keys_by_plaintext.remove(&plaintext);
|
||||
}
|
||||
drop(cache);
|
||||
self.pending_last_used
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(id);
|
||||
}
|
||||
|
||||
fn queue_last_used_touch(&self, id: String) {
|
||||
self.pending_last_used
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(id, Utc::now().to_rfc3339());
|
||||
self.schedule_flush();
|
||||
}
|
||||
|
||||
fn schedule_flush(&self) {
|
||||
if self
|
||||
.flush_scheduled
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let runtime = self.clone();
|
||||
tokio::spawn(async move {
|
||||
sleep(runtime.flush_delay).await;
|
||||
runtime.flush_pending_last_used().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// 立即刷写待持久化的最近使用时间。
|
||||
pub async fn flush_pending_last_used(&self) {
|
||||
let updates = self.take_pending_last_used();
|
||||
let mut failed = BTreeMap::new();
|
||||
for (id, last_used_at) in updates {
|
||||
if let Err(error) = self.store.update_last_used_at(&id, &last_used_at).await {
|
||||
tracing::warn!(
|
||||
key_id = %id,
|
||||
error = %error,
|
||||
"failed to flush client api key last_used_at"
|
||||
);
|
||||
failed.insert(id, last_used_at);
|
||||
}
|
||||
}
|
||||
if !failed.is_empty() {
|
||||
self.pending_last_used
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.extend(failed);
|
||||
}
|
||||
self.flush_scheduled.store(false, Ordering::Release);
|
||||
if !self
|
||||
.pending_last_used
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.is_empty()
|
||||
{
|
||||
self.schedule_flush();
|
||||
}
|
||||
}
|
||||
|
||||
fn take_pending_last_used(&self) -> BTreeMap<String, String> {
|
||||
std::mem::take(
|
||||
&mut *self
|
||||
.pending_last_used
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn map_client_key_store_error(error: &SqliteClientKeyStoreError) -> ClientKeyStoreError {
|
||||
ClientKeyStoreError::OperationFailed {
|
||||
message: error.to_string(),
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
//! 管理端 HTTP API。
|
||||
|
||||
/// 账号 CRUD 处理器。
|
||||
pub mod accounts;
|
||||
/// 管理端账号认证状态与本地管理员会话处理器。
|
||||
pub mod auth;
|
||||
/// v1 接口访问 Key 处理器。
|
||||
pub mod keys;
|
||||
/// 管理面日志、用量与诊断统计。
|
||||
pub mod monitoring;
|
||||
/// 响应封装。
|
||||
pub mod response;
|
||||
/// 管理端路由。
|
||||
pub mod router;
|
||||
/// 设置处理器。
|
||||
pub mod settings;
|
||||
/// 系统版本与在线更新处理器。
|
||||
pub mod system;
|
||||
pub(crate) mod update_payload;
|
||||
@@ -1,382 +0,0 @@
|
||||
//! SQLite 用量存储。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite, SqlitePool};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::infra::{
|
||||
json::{decode_cursor, encode_cursor, Page},
|
||||
time::{
|
||||
parse_optional_rfc3339_utc as parse_optional_rfc3339, parse_rfc3339_utc as parse_rfc3339,
|
||||
},
|
||||
};
|
||||
|
||||
const LIST_USAGE_SELECT_SQL: &str = r"
|
||||
select
|
||||
au.account_id,
|
||||
a.email,
|
||||
a.label,
|
||||
a.plan_type,
|
||||
au.request_count,
|
||||
au.empty_response_count,
|
||||
au.input_tokens,
|
||||
au.output_tokens,
|
||||
au.cached_tokens,
|
||||
au.reasoning_tokens,
|
||||
au.total_tokens,
|
||||
au.image_input_tokens,
|
||||
au.image_output_tokens,
|
||||
au.image_request_count,
|
||||
au.image_request_failed_count,
|
||||
au.window_request_count,
|
||||
au.window_input_tokens,
|
||||
au.window_output_tokens,
|
||||
au.window_cached_tokens,
|
||||
au.window_started_at,
|
||||
au.window_reset_at,
|
||||
au.limit_window_seconds,
|
||||
au.last_used_at
|
||||
from account_usage au
|
||||
left join accounts a on a.id = au.account_id";
|
||||
|
||||
const USAGE_SUMMARY_SQL: &str = r"
|
||||
select
|
||||
count(*) as account_count,
|
||||
coalesce(sum(request_count), 0) as request_count,
|
||||
coalesce(sum(empty_response_count), 0) as empty_response_count,
|
||||
coalesce(sum(input_tokens), 0) as input_tokens,
|
||||
coalesce(sum(output_tokens), 0) as output_tokens,
|
||||
coalesce(sum(cached_tokens), 0) as cached_tokens,
|
||||
coalesce(sum(reasoning_tokens), 0) as reasoning_tokens,
|
||||
coalesce(sum(total_tokens), 0) as total_tokens,
|
||||
coalesce(sum(image_input_tokens), 0) as image_input_tokens,
|
||||
coalesce(sum(image_output_tokens), 0) as image_output_tokens,
|
||||
coalesce(sum(image_request_count), 0) as image_request_count,
|
||||
coalesce(sum(image_request_failed_count), 0) as image_request_failed_count
|
||||
from account_usage";
|
||||
|
||||
const LIST_USAGE_TIME_BUCKETS_SQL: &str = r"
|
||||
select
|
||||
bucket_start,
|
||||
model,
|
||||
service_tier,
|
||||
coalesce(sum(request_count), 0) as request_count,
|
||||
coalesce(sum(error_count), 0) as error_count,
|
||||
coalesce(sum(input_tokens), 0) as input_tokens,
|
||||
coalesce(sum(output_tokens), 0) as output_tokens,
|
||||
coalesce(sum(cached_tokens), 0) as cached_tokens,
|
||||
coalesce(sum(first_token_latency_sum), 0) as first_token_latency_sum,
|
||||
coalesce(sum(first_token_latency_count), 0) as first_token_latency_count,
|
||||
coalesce(sum(latency_sum), 0) as latency_sum,
|
||||
coalesce(sum(latency_count), 0) as latency_count,
|
||||
coalesce(max(max_latency_ms), 0) as max_latency_ms,
|
||||
coalesce(min(nullif(min_latency_ms, 0)), 0) as min_latency_ms
|
||||
from usage_time_buckets
|
||||
where bucket_start >= ? and bucket_start <= ?
|
||||
group by bucket_start, model, service_tier
|
||||
order by bucket_start asc, model asc, service_tier asc";
|
||||
|
||||
/// SQLite 用量存储错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SqliteUsageStoreError {
|
||||
/// 数据库错误。
|
||||
#[error("sqlite usage store database error: {0}")]
|
||||
Database(#[from] sqlx::Error),
|
||||
/// 时间格式错误。
|
||||
#[error("sqlite usage store timestamp error: {0}")]
|
||||
Timestamp(#[from] chrono::ParseError),
|
||||
/// 分页游标非法。
|
||||
#[error("invalid usage pagination cursor")]
|
||||
InvalidCursor,
|
||||
}
|
||||
|
||||
/// SQLite 用量存储结果。
|
||||
pub type SqliteUsageStoreResult<T> = Result<T, SqliteUsageStoreError>;
|
||||
|
||||
/// 账号用量列表记录。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UsageListRecord {
|
||||
/// 账号 ID。
|
||||
pub account_id: String,
|
||||
/// 邮箱。
|
||||
pub email: Option<String>,
|
||||
/// 标签。
|
||||
pub label: Option<String>,
|
||||
/// 计划类型。
|
||||
pub plan_type: Option<String>,
|
||||
/// 历史请求总数。
|
||||
pub request_count: i64,
|
||||
/// 历史空响应次数。
|
||||
pub empty_response_count: i64,
|
||||
/// 累计输入 token。
|
||||
pub input_tokens: i64,
|
||||
/// 累计输出 token。
|
||||
pub output_tokens: i64,
|
||||
/// 累计缓存 token。
|
||||
pub cached_tokens: i64,
|
||||
/// 累计 reasoning token。
|
||||
pub reasoning_tokens: i64,
|
||||
/// 累计总 token。
|
||||
pub total_tokens: i64,
|
||||
/// 累计图片输入 token。
|
||||
pub image_input_tokens: i64,
|
||||
/// 累计图片输出 token。
|
||||
pub image_output_tokens: i64,
|
||||
/// 累计图片请求成功次数。
|
||||
pub image_request_count: i64,
|
||||
/// 累计图片请求失败次数。
|
||||
pub image_request_failed_count: i64,
|
||||
/// 当前额度窗口请求数。
|
||||
pub window_request_count: i64,
|
||||
/// 当前额度窗口输入 token。
|
||||
pub window_input_tokens: i64,
|
||||
/// 当前额度窗口输出 token。
|
||||
pub window_output_tokens: i64,
|
||||
/// 当前额度窗口缓存 token。
|
||||
pub window_cached_tokens: i64,
|
||||
/// 当前额度窗口开始时间。
|
||||
pub window_started_at: Option<DateTime<Utc>>,
|
||||
/// 当前额度窗口重置时间。
|
||||
pub window_reset_at: Option<DateTime<Utc>>,
|
||||
/// 当前额度窗口大小(秒)。
|
||||
pub limit_window_seconds: Option<u64>,
|
||||
/// 最近使用时间。
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 账号用量汇总。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UsageSummary {
|
||||
/// 账号数。
|
||||
pub account_count: i64,
|
||||
/// 总请求数。
|
||||
pub request_count: i64,
|
||||
/// 总空响应数。
|
||||
pub empty_response_count: i64,
|
||||
/// 总输入 token。
|
||||
pub input_tokens: i64,
|
||||
/// 总输出 token。
|
||||
pub output_tokens: i64,
|
||||
/// 总缓存 token。
|
||||
pub cached_tokens: i64,
|
||||
/// 总 reasoning token。
|
||||
pub reasoning_tokens: i64,
|
||||
/// 总 token。
|
||||
pub total_tokens: i64,
|
||||
/// 总图片输入 token。
|
||||
pub image_input_tokens: i64,
|
||||
/// 总图片输出 token。
|
||||
pub image_output_tokens: i64,
|
||||
/// 总图片请求成功数。
|
||||
pub image_request_count: i64,
|
||||
/// 总图片请求失败数。
|
||||
pub image_request_failed_count: i64,
|
||||
}
|
||||
|
||||
/// 时间桶聚合用量记录。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UsageTimeBucketRecord {
|
||||
/// 桶开始时间。
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
/// 模型。
|
||||
pub model: String,
|
||||
/// 服务层级。
|
||||
pub service_tier: Option<String>,
|
||||
/// 请求数。
|
||||
pub request_count: i64,
|
||||
/// 错误数。
|
||||
pub error_count: i64,
|
||||
/// 输入 token。
|
||||
pub input_tokens: i64,
|
||||
/// 输出 token。
|
||||
pub output_tokens: i64,
|
||||
/// 缓存 token。
|
||||
pub cached_tokens: i64,
|
||||
/// 首 token 延迟总和。
|
||||
pub first_token_latency_sum: i64,
|
||||
/// 首 token 延迟样本数。
|
||||
pub first_token_latency_count: i64,
|
||||
/// 完成延迟总和。
|
||||
pub latency_sum: i64,
|
||||
/// 完成延迟样本数。
|
||||
pub latency_count: i64,
|
||||
/// 最大完成延迟。
|
||||
pub max_latency_ms: i64,
|
||||
/// 最小完成延迟。
|
||||
pub min_latency_ms: i64,
|
||||
}
|
||||
|
||||
/// SQLite 用量存储。
|
||||
#[derive(Clone)]
|
||||
pub struct SqliteUsageStore {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SqliteUsageStore {
|
||||
/// 构造存储。
|
||||
pub fn new(pool: SqlitePool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// 分页列出账号用量。
|
||||
pub async fn list_usage(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: u32,
|
||||
) -> SqliteUsageStoreResult<Page<UsageListRecord>> {
|
||||
let limit = limit.clamp(1, 200);
|
||||
let fetch_limit = i64::from(limit) + 1;
|
||||
let mut builder = QueryBuilder::<Sqlite>::new(LIST_USAGE_SELECT_SQL);
|
||||
if let Some(cursor) = cursor {
|
||||
let (last_used_at, account_id) =
|
||||
decode_cursor(&cursor).ok_or(SqliteUsageStoreError::InvalidCursor)?;
|
||||
builder.push("\nwhere au.last_used_at < ");
|
||||
builder.push_bind(last_used_at.clone());
|
||||
builder.push("\n or (au.last_used_at = ");
|
||||
builder.push_bind(last_used_at);
|
||||
builder.push(" and au.account_id < ");
|
||||
builder.push_bind(account_id);
|
||||
builder.push(")");
|
||||
}
|
||||
builder.push("\norder by au.last_used_at desc, au.account_id desc\nlimit ");
|
||||
builder.push_bind(fetch_limit);
|
||||
|
||||
let rows = builder.build().fetch_all(&self.pool).await?;
|
||||
Ok(to_page(&rows, limit))
|
||||
}
|
||||
|
||||
/// 按账号 ID 批量读取账号用量。
|
||||
pub async fn list_usage_by_account_ids(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> SqliteUsageStoreResult<Vec<UsageListRecord>> {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut builder = QueryBuilder::<Sqlite>::new(LIST_USAGE_SELECT_SQL);
|
||||
builder.push("\nwhere au.account_id in (");
|
||||
let mut separated = builder.separated(", ");
|
||||
for account_id in account_ids {
|
||||
separated.push_bind(account_id);
|
||||
}
|
||||
separated.push_unseparated(")");
|
||||
|
||||
let rows = builder.build().fetch_all(&self.pool).await?;
|
||||
rows.iter().map(usage_list_from_row).collect()
|
||||
}
|
||||
|
||||
/// 汇总账号用量。
|
||||
pub async fn usage_summary(&self) -> SqliteUsageStoreResult<UsageSummary> {
|
||||
let row = sqlx::query(USAGE_SUMMARY_SQL).fetch_one(&self.pool).await?;
|
||||
Ok(UsageSummary {
|
||||
account_count: row.get("account_count"),
|
||||
request_count: row.get("request_count"),
|
||||
empty_response_count: row.get("empty_response_count"),
|
||||
input_tokens: row.get("input_tokens"),
|
||||
output_tokens: row.get("output_tokens"),
|
||||
cached_tokens: row.get("cached_tokens"),
|
||||
reasoning_tokens: row.get("reasoning_tokens"),
|
||||
total_tokens: row.get("total_tokens"),
|
||||
image_input_tokens: row.get("image_input_tokens"),
|
||||
image_output_tokens: row.get("image_output_tokens"),
|
||||
image_request_count: row.get("image_request_count"),
|
||||
image_request_failed_count: row.get("image_request_failed_count"),
|
||||
})
|
||||
}
|
||||
|
||||
/// 列出指定时间范围内的时间桶聚合用量。
|
||||
pub async fn list_time_buckets(
|
||||
&self,
|
||||
start: DateTime<Utc>,
|
||||
end: DateTime<Utc>,
|
||||
) -> SqliteUsageStoreResult<Vec<UsageTimeBucketRecord>> {
|
||||
let rows = sqlx::query(LIST_USAGE_TIME_BUCKETS_SQL)
|
||||
.bind(start.to_rfc3339())
|
||||
.bind(end.to_rfc3339())
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
rows.iter().map(usage_time_bucket_from_row).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_list_from_row(row: &SqliteRow) -> SqliteUsageStoreResult<UsageListRecord> {
|
||||
Ok(UsageListRecord {
|
||||
account_id: row.get("account_id"),
|
||||
email: row.get("email"),
|
||||
label: row.get("label"),
|
||||
plan_type: row.get("plan_type"),
|
||||
request_count: row.get("request_count"),
|
||||
empty_response_count: row.get("empty_response_count"),
|
||||
input_tokens: row.get("input_tokens"),
|
||||
output_tokens: row.get("output_tokens"),
|
||||
cached_tokens: row.get("cached_tokens"),
|
||||
reasoning_tokens: row.get("reasoning_tokens"),
|
||||
total_tokens: row.get("total_tokens"),
|
||||
image_input_tokens: row.get("image_input_tokens"),
|
||||
image_output_tokens: row.get("image_output_tokens"),
|
||||
image_request_count: row.get("image_request_count"),
|
||||
image_request_failed_count: row.get("image_request_failed_count"),
|
||||
window_request_count: row.get("window_request_count"),
|
||||
window_input_tokens: row.get("window_input_tokens"),
|
||||
window_output_tokens: row.get("window_output_tokens"),
|
||||
window_cached_tokens: row.get("window_cached_tokens"),
|
||||
window_started_at: parse_optional_rfc3339(
|
||||
row.get::<Option<String>, _>("window_started_at").as_deref(),
|
||||
)?,
|
||||
window_reset_at: parse_optional_rfc3339(
|
||||
row.get::<Option<String>, _>("window_reset_at").as_deref(),
|
||||
)?,
|
||||
limit_window_seconds: row
|
||||
.get::<Option<i64>, _>("limit_window_seconds")
|
||||
.and_then(|value| u64::try_from(value).ok()),
|
||||
last_used_at: parse_optional_rfc3339(
|
||||
row.get::<Option<String>, _>("last_used_at").as_deref(),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn usage_time_bucket_from_row(row: &SqliteRow) -> SqliteUsageStoreResult<UsageTimeBucketRecord> {
|
||||
let service_tier = row.get::<String, _>("service_tier").trim().to_string();
|
||||
Ok(UsageTimeBucketRecord {
|
||||
bucket_start: parse_rfc3339(row.get::<&str, _>("bucket_start"))?,
|
||||
model: row.get("model"),
|
||||
service_tier: (!service_tier.is_empty()).then_some(service_tier),
|
||||
request_count: row.get("request_count"),
|
||||
error_count: row.get("error_count"),
|
||||
input_tokens: row.get("input_tokens"),
|
||||
output_tokens: row.get("output_tokens"),
|
||||
cached_tokens: row.get("cached_tokens"),
|
||||
first_token_latency_sum: row.get("first_token_latency_sum"),
|
||||
first_token_latency_count: row.get("first_token_latency_count"),
|
||||
latency_sum: row.get("latency_sum"),
|
||||
latency_count: row.get("latency_count"),
|
||||
max_latency_ms: row.get("max_latency_ms"),
|
||||
min_latency_ms: row.get("min_latency_ms"),
|
||||
})
|
||||
}
|
||||
|
||||
fn to_page(rows: &[SqliteRow], limit: u32) -> Page<UsageListRecord> {
|
||||
let has_more = rows.len() > limit as usize;
|
||||
let mut items = Vec::with_capacity(limit as usize);
|
||||
let mut last_row: Option<&SqliteRow> = None;
|
||||
for (i, row) in rows.iter().enumerate() {
|
||||
if i >= limit as usize {
|
||||
break;
|
||||
}
|
||||
if let Ok(item) = usage_list_from_row(row) {
|
||||
items.push(item);
|
||||
last_row = Some(row);
|
||||
}
|
||||
}
|
||||
let next_cursor = if has_more {
|
||||
last_row.map(|row| {
|
||||
let ts: String = row.get("last_used_at");
|
||||
let id: String = row.get("account_id");
|
||||
encode_cursor(&ts, &id)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Page { items, next_cursor }
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,90 +0,0 @@
|
||||
//! 诊断数据类型与聚合函数。
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{upstream::accounts::pool::AccountCapacitySummary, upstream::fingerprint::Fingerprint};
|
||||
|
||||
/// 账号池摘要。
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountPoolDiagnostics {
|
||||
/// 总数。
|
||||
pub total: usize,
|
||||
/// 活跃数。
|
||||
pub active: usize,
|
||||
/// 过期数。
|
||||
pub expired: usize,
|
||||
/// 配额耗尽数。
|
||||
pub quota_exhausted: usize,
|
||||
/// 刷新中数。
|
||||
pub refreshing: usize,
|
||||
/// 禁用数。
|
||||
pub disabled: usize,
|
||||
/// 封禁数。
|
||||
pub banned: usize,
|
||||
}
|
||||
|
||||
/// Account-pool capacity diagnostics.
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountCapacityDiagnostics {
|
||||
/// Maximum concurrent slots per account.
|
||||
pub max_concurrent_per_account: usize,
|
||||
/// Total slots available across active accounts.
|
||||
pub total_slots: usize,
|
||||
/// Currently occupied slots.
|
||||
pub used_slots: usize,
|
||||
/// Currently available slots.
|
||||
pub available_slots: usize,
|
||||
}
|
||||
|
||||
/// Runtime fingerprint summary.
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FingerprintDiagnostics {
|
||||
/// Fingerprint source label.
|
||||
pub source: &'static str,
|
||||
/// Client originator.
|
||||
pub originator: String,
|
||||
/// App version.
|
||||
pub app_version: String,
|
||||
/// Build number.
|
||||
pub build_number: String,
|
||||
/// Platform name.
|
||||
pub platform: String,
|
||||
/// Architecture name.
|
||||
pub arch: String,
|
||||
/// Chromium major version.
|
||||
pub chromium_version: String,
|
||||
/// Expanded user-agent.
|
||||
pub user_agent: String,
|
||||
/// 指纹最后更新时间。
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub updated_at: Option<String>,
|
||||
}
|
||||
|
||||
/// 构造 fingerprint 诊断数据。
|
||||
pub fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics {
|
||||
FingerprintDiagnostics {
|
||||
source: "runtime",
|
||||
originator: fingerprint.originator.clone(),
|
||||
app_version: fingerprint.app_version.clone(),
|
||||
build_number: fingerprint.build_number.clone(),
|
||||
platform: fingerprint.platform.clone(),
|
||||
arch: fingerprint.arch.clone(),
|
||||
chromium_version: fingerprint.chromium_version.clone(),
|
||||
user_agent: fingerprint.user_agent(),
|
||||
updated_at: fingerprint.updated_at.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AccountCapacitySummary> for AccountCapacityDiagnostics {
|
||||
fn from(summary: AccountCapacitySummary) -> Self {
|
||||
Self {
|
||||
max_concurrent_per_account: summary.max_concurrent_per_account,
|
||||
total_slots: summary.total_slots,
|
||||
used_slots: summary.used_slots,
|
||||
available_slots: summary.available_slots,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
//! 管理面监控模块:日志、用量统计和 Dashboard 聚合。
|
||||
|
||||
pub mod account_usage_service;
|
||||
pub mod account_usage_store;
|
||||
pub(crate) mod billing;
|
||||
pub mod dashboard;
|
||||
pub mod diagnostics;
|
||||
pub mod usage_record_model;
|
||||
pub mod usage_record_routes;
|
||||
pub mod usage_record_service;
|
||||
pub mod usage_record_store;
|
||||
@@ -1,355 +0,0 @@
|
||||
//! 管理端使用记录服务。
|
||||
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{
|
||||
admin::monitoring::{
|
||||
billing,
|
||||
usage_record_model::metadata_service_tier,
|
||||
usage_record_model::{UsageRecord, UsageRecordLevel},
|
||||
usage_record_store::{
|
||||
SqliteUsageRecordStore, UsageRecordBreakdown, UsageRecordEndpointSource,
|
||||
UsageRecordFilter, UsageRecordModelSource, UsageRecordSummary, UsageRecordTrendPoint,
|
||||
},
|
||||
},
|
||||
infra::json::{NumberedPage, Page},
|
||||
};
|
||||
|
||||
/// 管理端使用记录查询过滤器。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AdminUsageRecordFilter {
|
||||
/// 事件类别。
|
||||
pub kind: Option<String>,
|
||||
/// 事件等级。
|
||||
pub level: Option<UsageRecordLevel>,
|
||||
/// 请求 ID。
|
||||
pub request_id: Option<String>,
|
||||
/// 账号 ID。
|
||||
pub account_id: Option<String>,
|
||||
/// 路由。
|
||||
pub route: Option<String>,
|
||||
/// 模型。
|
||||
pub model: Option<String>,
|
||||
/// HTTP 状态码。
|
||||
pub status_code: Option<i64>,
|
||||
/// 上游传输方式。
|
||||
pub transport: Option<String>,
|
||||
/// 同一请求内的上游尝试序号。
|
||||
pub attempt_index: Option<i64>,
|
||||
/// 上游 HTTP 状态码。
|
||||
pub upstream_status_code: Option<i64>,
|
||||
/// 失败分类。
|
||||
pub failure_class: Option<String>,
|
||||
/// 上游响应 ID。
|
||||
pub response_id: Option<String>,
|
||||
/// 上游请求 ID。
|
||||
pub upstream_request_id: Option<String>,
|
||||
/// 搜索关键词。
|
||||
pub search: Option<String>,
|
||||
/// 起始时间。
|
||||
pub start_time: Option<DateTime<Utc>>,
|
||||
/// 结束时间。
|
||||
pub end_time: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl From<AdminUsageRecordFilter> for UsageRecordFilter {
|
||||
fn from(filter: AdminUsageRecordFilter) -> Self {
|
||||
Self {
|
||||
kind: filter.kind,
|
||||
level: filter.level,
|
||||
request_id: filter.request_id,
|
||||
account_id: filter.account_id,
|
||||
route: filter.route,
|
||||
model: filter.model,
|
||||
status_code: filter.status_code,
|
||||
transport: filter.transport,
|
||||
attempt_index: filter.attempt_index,
|
||||
upstream_status_code: filter.upstream_status_code,
|
||||
failure_class: filter.failure_class,
|
||||
response_id: filter.response_id,
|
||||
upstream_request_id: filter.upstream_request_id,
|
||||
search: filter.search,
|
||||
start_time: filter.start_time,
|
||||
end_time: filter.end_time,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 清空使用记录结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AdminClearUsageRecords {
|
||||
/// 清理数量。
|
||||
pub cleared: u64,
|
||||
}
|
||||
|
||||
/// 管理端使用记录费用明细。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct AdminUsageRecordCostDetails {
|
||||
pub input_cost: f64,
|
||||
pub output_cost: f64,
|
||||
pub cache_read_cost: f64,
|
||||
pub total_cost: f64,
|
||||
pub original_cost: f64,
|
||||
pub input_price_per_mtoken: f64,
|
||||
pub output_price_per_mtoken: f64,
|
||||
pub cache_read_price_per_mtoken: f64,
|
||||
pub service_tier: Option<String>,
|
||||
pub service_tier_display: String,
|
||||
pub multiplier: f64,
|
||||
}
|
||||
|
||||
/// 管理端使用记录错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AdminUsageRecordError {
|
||||
/// 列表失败。
|
||||
#[error("failed to list usage records")]
|
||||
List,
|
||||
/// 读取失败。
|
||||
#[error("failed to get usage record")]
|
||||
Get,
|
||||
/// 清空失败。
|
||||
#[error("failed to clear usage records")]
|
||||
Clear,
|
||||
/// 写入失败。
|
||||
#[error("failed to append usage record")]
|
||||
Append,
|
||||
/// 裁剪失败。
|
||||
#[error("failed to trim usage records")]
|
||||
Trim,
|
||||
/// 账号关联信息读取失败。
|
||||
#[error("failed to load usage record accounts")]
|
||||
Accounts,
|
||||
}
|
||||
|
||||
/// 管理端使用记录服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AdminUsageRecordService {
|
||||
store: SqliteUsageRecordStore,
|
||||
settings: Arc<tokio::sync::RwLock<AdminUsageRecordSettings>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct AdminUsageRecordSettings {
|
||||
enabled: bool,
|
||||
capacity: u32,
|
||||
capture_body: bool,
|
||||
}
|
||||
|
||||
impl AdminUsageRecordService {
|
||||
/// 构造管理端使用记录服务。
|
||||
pub fn new(
|
||||
store: SqliteUsageRecordStore,
|
||||
enabled: bool,
|
||||
capacity: u32,
|
||||
capture_body: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
settings: Arc::new(tokio::sync::RwLock::new(AdminUsageRecordSettings {
|
||||
enabled,
|
||||
capacity,
|
||||
capture_body,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// 分页查询日志。
|
||||
pub async fn list(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: u32,
|
||||
filter: AdminUsageRecordFilter,
|
||||
) -> Result<Page<UsageRecord>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.list(filter.into(), cursor, limit)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 按页码查询日志。
|
||||
pub async fn list_page(
|
||||
&self,
|
||||
page: u32,
|
||||
page_size: u32,
|
||||
filter: AdminUsageRecordFilter,
|
||||
) -> Result<NumberedPage<UsageRecord>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.list_page(filter.into(), page, page_size)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 按 ID 读取日志。
|
||||
pub async fn get(&self, id: &str) -> Result<Option<UsageRecord>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Get)
|
||||
}
|
||||
|
||||
/// 读取使用记录关联账号的邮箱映射。
|
||||
pub async fn account_email_map(
|
||||
&self,
|
||||
items: &[UsageRecord],
|
||||
) -> Result<HashMap<String, String>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.account_email_map(items)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Accounts)
|
||||
}
|
||||
|
||||
/// 汇总使用记录。
|
||||
pub async fn summary(
|
||||
&self,
|
||||
filter: AdminUsageRecordFilter,
|
||||
) -> Result<UsageRecordSummary, AdminUsageRecordError> {
|
||||
self.store
|
||||
.summary(filter.into())
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 按模型来源聚合使用记录分布。
|
||||
pub async fn model_distribution(
|
||||
&self,
|
||||
filter: AdminUsageRecordFilter,
|
||||
source: UsageRecordModelSource,
|
||||
) -> Result<Vec<UsageRecordBreakdown>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.model_distribution(filter.into(), source)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 按端点来源聚合使用记录分布。
|
||||
pub async fn endpoint_distribution(
|
||||
&self,
|
||||
filter: AdminUsageRecordFilter,
|
||||
source: UsageRecordEndpointSource,
|
||||
) -> Result<Vec<UsageRecordBreakdown>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.endpoint_distribution(filter.into(), source)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 聚合使用记录趋势。
|
||||
pub async fn trend(
|
||||
&self,
|
||||
filter: AdminUsageRecordFilter,
|
||||
) -> Result<Vec<UsageRecordTrendPoint>, AdminUsageRecordError> {
|
||||
self.store
|
||||
.trend(filter.into())
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::List)
|
||||
}
|
||||
|
||||
/// 清空日志。
|
||||
pub async fn clear(&self) -> Result<AdminClearUsageRecords, AdminUsageRecordError> {
|
||||
self.store
|
||||
.clear()
|
||||
.await
|
||||
.map(|cleared| AdminClearUsageRecords { cleared })
|
||||
.map_err(|_| AdminUsageRecordError::Clear)
|
||||
}
|
||||
|
||||
/// 记录使用记录。
|
||||
pub async fn record(&self, event: UsageRecord) -> Result<(), AdminUsageRecordError> {
|
||||
let settings = *self.settings.read().await;
|
||||
if !settings.enabled && event.level != UsageRecordLevel::Error {
|
||||
return Ok(());
|
||||
}
|
||||
self.append_with_settings(event, settings).await
|
||||
}
|
||||
|
||||
async fn append_with_settings(
|
||||
&self,
|
||||
mut event: UsageRecord,
|
||||
settings: AdminUsageRecordSettings,
|
||||
) -> Result<(), AdminUsageRecordError> {
|
||||
apply_capture_body_policy(&mut event, settings.capture_body);
|
||||
self.store
|
||||
.append(&event)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Append)?;
|
||||
self.store
|
||||
.trim_to_capacity(settings.capacity)
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Trim)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_capture_body_policy(event: &mut UsageRecord, capture_body: bool) {
|
||||
if capture_body {
|
||||
return;
|
||||
}
|
||||
let Some(metadata) = event.metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
for key in [
|
||||
"body",
|
||||
"rawBody",
|
||||
"requestBody",
|
||||
"responseBody",
|
||||
"upstreamBody",
|
||||
] {
|
||||
metadata.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_record_cost_details(
|
||||
record: &UsageRecord,
|
||||
upstream_model: Option<&str>,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cached_tokens: u64,
|
||||
) -> Option<AdminUsageRecordCostDetails> {
|
||||
if input_tokens == 0 && output_tokens == 0 && cached_tokens == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let model = upstream_model
|
||||
.or(record.model.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let service_tier = metadata_service_tier(&record.metadata);
|
||||
let breakdown = billing::calculate_cost_breakdown(
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cached_tokens,
|
||||
model,
|
||||
service_tier,
|
||||
);
|
||||
let original_cost = breakdown.input_cost + breakdown.output_cost + breakdown.cache_read_cost;
|
||||
|
||||
Some(AdminUsageRecordCostDetails {
|
||||
input_cost: breakdown.input_cost,
|
||||
output_cost: breakdown.output_cost,
|
||||
cache_read_cost: breakdown.cache_read_cost,
|
||||
total_cost: breakdown.total_cost,
|
||||
original_cost,
|
||||
input_price_per_mtoken: breakdown.input_price_per_mtoken,
|
||||
output_price_per_mtoken: breakdown.output_price_per_mtoken,
|
||||
cache_read_price_per_mtoken: breakdown.cache_read_price_per_mtoken,
|
||||
service_tier: breakdown.service_tier.clone(),
|
||||
service_tier_display: breakdown
|
||||
.service_tier
|
||||
.as_deref()
|
||||
.map(format_service_tier)
|
||||
.unwrap_or_else(|| "Default".to_string()),
|
||||
multiplier: breakdown.tier_multiplier,
|
||||
})
|
||||
}
|
||||
|
||||
fn format_service_tier(value: &str) -> String {
|
||||
match value {
|
||||
"priority" | "fast" => "Fast".to_string(),
|
||||
"flex" => "Flex".to_string(),
|
||||
"default" => "Default".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +0,0 @@
|
||||
//! 管理端设置 HTTP 模块。
|
||||
|
||||
pub mod routes;
|
||||
@@ -1,5 +0,0 @@
|
||||
//! 系统版本与在线更新处理器。
|
||||
|
||||
pub mod routes;
|
||||
mod state;
|
||||
mod updater;
|
||||
@@ -1,5 +0,0 @@
|
||||
//! 管理端系统路由入口。
|
||||
|
||||
pub(crate) use super::updater::{
|
||||
perform_update, restart, rollback, update_detail, update_event_stream, update_status, version,
|
||||
};
|
||||
@@ -1,780 +0,0 @@
|
||||
//! 管理端系统版本与在线更新路由。
|
||||
|
||||
use std::{
|
||||
convert::Infallible,
|
||||
env, fmt, fs, io,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
sync::OnceLock,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{Json, Query},
|
||||
http::StatusCode,
|
||||
response::{
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
IntoResponse,
|
||||
},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::Stream;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::{broadcast, Mutex};
|
||||
|
||||
use crate::{
|
||||
admin::auth::session::AdminAuth,
|
||||
admin::response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
runtime::shutdown::{request_process_restart, request_shutdown, shutdown_subscription},
|
||||
};
|
||||
|
||||
use super::state::{
|
||||
finish_operation, operation_id, read_state, set_operation_running, OperationLock,
|
||||
SystemOperationKind,
|
||||
};
|
||||
|
||||
mod download;
|
||||
use download::{
|
||||
download_file, validate_download_url, validate_github_api_base, verify_checksum,
|
||||
DownloadProgress,
|
||||
};
|
||||
mod archive;
|
||||
use archive::{extract_release_archive, replace_release_files, rollback_release_update};
|
||||
mod release;
|
||||
use release::{
|
||||
check_latest_release, fetch_latest_release, normalize_version_tag, select_release_archive,
|
||||
update_info_from_release, GitHubRelease, UpdateInfoData,
|
||||
};
|
||||
|
||||
const APP_BINARY_NAME: &str = "codex-proxy-rs";
|
||||
const DEFAULT_WEB_DIST_DIR: &str = "/app/web/dist";
|
||||
const GITHUB_API_BASE: &str = "https://api.github.com/repos";
|
||||
const MAX_DOWNLOAD_SIZE: u64 = 500 * 1024 * 1024;
|
||||
const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS";
|
||||
const REPLACEMENT_START_DELAY_MS: &str = "1200";
|
||||
|
||||
static SYSTEM_OPERATION_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
static UPDATE_EVENT_SENDER: OnceLock<broadcast::Sender<SystemUpdateEvent>> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UpdateDetailQuery {
|
||||
refresh: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UpdateRequest {
|
||||
target_version: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct VersionData {
|
||||
version: String,
|
||||
git_sha: String,
|
||||
build_time: String,
|
||||
deployment_mode: String,
|
||||
deployment_mode_label: String,
|
||||
update_channel: String,
|
||||
latest_version: String,
|
||||
has_update: bool,
|
||||
update_cached: bool,
|
||||
update_warning: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UpdateStartedData {
|
||||
operation_id: String,
|
||||
deployment_mode: String,
|
||||
message: String,
|
||||
need_restart: bool,
|
||||
target_version: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum UpdateLogLevel {
|
||||
Info,
|
||||
Success,
|
||||
Warning,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SystemUpdateEvent {
|
||||
id: String,
|
||||
operation_id: Option<String>,
|
||||
level: UpdateLogLevel,
|
||||
step: Option<String>,
|
||||
message: String,
|
||||
at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct SystemUpdateConfig {
|
||||
version: String,
|
||||
git_sha: String,
|
||||
build_time: String,
|
||||
deployment_mode: String,
|
||||
build_type: String,
|
||||
update_channel: String,
|
||||
update_repository: Option<String>,
|
||||
github_api_base: String,
|
||||
executable_path: Option<PathBuf>,
|
||||
web_dist_dir: PathBuf,
|
||||
update_state_file: PathBuf,
|
||||
update_lock_file: PathBuf,
|
||||
update_temp_dir: PathBuf,
|
||||
}
|
||||
|
||||
/// `GET /api/admin/system/version`
|
||||
pub(crate) async fn version(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
let update_info = check_latest_release(&config, false).await;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(config.version_data(&update_info)),
|
||||
))
|
||||
}
|
||||
|
||||
/// `GET /api/admin/system/update-detail`
|
||||
pub(crate) async fn update_detail(
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<UpdateDetailQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
let info = check_latest_release(&config, query.refresh.unwrap_or(false)).await;
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(info)))
|
||||
}
|
||||
|
||||
/// `GET /api/admin/system/update-events`
|
||||
pub(crate) async fn update_event_stream(
|
||||
_auth: AdminAuth,
|
||||
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, AdminError> {
|
||||
let receiver = update_event_sender().subscribe();
|
||||
let shutdown = shutdown_subscription();
|
||||
let stream = futures::stream::unfold(
|
||||
(receiver, shutdown),
|
||||
|(mut receiver, mut shutdown)| async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = shutdown.recv() => return None,
|
||||
received = receiver.recv() => {
|
||||
match received {
|
||||
Ok(message) => {
|
||||
let id = message.id.clone();
|
||||
let data = serde_json::to_string(&message).unwrap_or_else(|_| "{}".to_string());
|
||||
let event = Event::default().event("update").id(id).data(data);
|
||||
return Some((Ok(event), (receiver, shutdown)));
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => return None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
|
||||
}
|
||||
|
||||
/// `POST /api/admin/system/update`
|
||||
pub(crate) async fn perform_update(
|
||||
_auth: AdminAuth,
|
||||
payload: Option<Json<UpdateRequest>>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let _guard = SYSTEM_OPERATION_LOCK
|
||||
.try_lock()
|
||||
.map_err(|_| AdminError::conflict("System update already running"))?;
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
if let Some(reason) = config.update_support_error() {
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Error,
|
||||
None,
|
||||
Some("preflight"),
|
||||
reason.clone(),
|
||||
);
|
||||
return Err(AdminError::conflict(reason));
|
||||
}
|
||||
let confirmed_target_version = confirmed_update_target(payload)?;
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
None,
|
||||
Some("release"),
|
||||
"正在获取最新 Release 信息",
|
||||
);
|
||||
let release = fetch_latest_release(
|
||||
&config.github_api_base,
|
||||
config
|
||||
.update_repository
|
||||
.as_deref()
|
||||
.ok_or_else(|| AdminError::conflict("Update checks require CPR_UPDATE_REPOSITORY"))?,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
emit_update_event(UpdateLogLevel::Error, None, Some("release"), error.clone());
|
||||
AdminError::bad_gateway(error)
|
||||
})?;
|
||||
let info = update_info_from_release(&config, release.clone());
|
||||
if info.latest_version != confirmed_target_version {
|
||||
let message = format!(
|
||||
"远端最新版本已变更为 v{},请重新检查并确认",
|
||||
info.latest_version
|
||||
);
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Warning,
|
||||
None,
|
||||
Some("release"),
|
||||
message.clone(),
|
||||
);
|
||||
return Err(AdminError::conflict(message));
|
||||
}
|
||||
if !info.has_update {
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Warning,
|
||||
None,
|
||||
Some("release"),
|
||||
"当前版本已是最新",
|
||||
);
|
||||
return Err(AdminError::conflict("Already up to date"));
|
||||
}
|
||||
let target_version = info.latest_version.clone();
|
||||
let operation_id = operation_id("update");
|
||||
let lock = OperationLock::acquire(&config.update_lock_file)?;
|
||||
set_operation_running(
|
||||
&config.update_state_file,
|
||||
&operation_id,
|
||||
SystemOperationKind::Update,
|
||||
Some(&target_version),
|
||||
)?;
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(&operation_id),
|
||||
Some("prepare"),
|
||||
format!("准备更新到 v{target_version}"),
|
||||
);
|
||||
let result = perform_release_update(&config, &release, &target_version, &operation_id).await;
|
||||
if let Err(error) = &result {
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Error,
|
||||
Some(&operation_id),
|
||||
Some("failed"),
|
||||
error.to_string(),
|
||||
);
|
||||
} else {
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Success,
|
||||
Some(&operation_id),
|
||||
Some("done"),
|
||||
"更新文件已替换,等待服务重启生效",
|
||||
);
|
||||
}
|
||||
finish_operation(
|
||||
&config.update_state_file,
|
||||
&operation_id,
|
||||
SystemOperationKind::Update,
|
||||
result.as_ref().ok().map(|_| target_version.clone()),
|
||||
result.as_ref().err().map(ToString::to_string),
|
||||
);
|
||||
drop(lock);
|
||||
result?;
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(UpdateStartedData {
|
||||
operation_id,
|
||||
deployment_mode: config.deployment_mode,
|
||||
message: "更新完成,请重启服务。".to_string(),
|
||||
need_restart: true,
|
||||
target_version,
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
fn confirmed_update_target(payload: Option<Json<UpdateRequest>>) -> Result<String, AdminError> {
|
||||
let Some(Json(payload)) = payload else {
|
||||
return Err(AdminError::conflict("更新前需要确认目标版本"));
|
||||
};
|
||||
let target_version = normalize_version_tag(&payload.target_version);
|
||||
if target_version.is_empty() {
|
||||
return Err(AdminError::bad_request("目标版本不能为空"));
|
||||
}
|
||||
Ok(target_version)
|
||||
}
|
||||
|
||||
/// `GET /api/admin/system/update-status`
|
||||
pub(crate) async fn update_status(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
let status = read_state(&config.update_state_file)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(status),
|
||||
))
|
||||
}
|
||||
|
||||
/// `POST /api/admin/system/rollback`
|
||||
pub(crate) async fn rollback(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
|
||||
let _guard = SYSTEM_OPERATION_LOCK
|
||||
.try_lock()
|
||||
.map_err(|_| AdminError::conflict("系统操作正在执行中"))?;
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
if let Some(reason) = config.update_support_error() {
|
||||
return Err(AdminError::conflict(reason));
|
||||
}
|
||||
|
||||
let operation_id = operation_id("rollback");
|
||||
let lock = OperationLock::acquire(&config.update_lock_file)?;
|
||||
set_operation_running(
|
||||
&config.update_state_file,
|
||||
&operation_id,
|
||||
SystemOperationKind::Rollback,
|
||||
None,
|
||||
)?;
|
||||
let result = rollback_release_update(&config).await;
|
||||
finish_operation(
|
||||
&config.update_state_file,
|
||||
&operation_id,
|
||||
SystemOperationKind::Rollback,
|
||||
None,
|
||||
result.as_ref().err().map(ToString::to_string),
|
||||
);
|
||||
drop(lock);
|
||||
result?;
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(serde_json::json!({
|
||||
"message": "回滚完成,请重启服务。",
|
||||
"needRestart": true,
|
||||
"operationId": operation_id
|
||||
})),
|
||||
))
|
||||
}
|
||||
|
||||
/// `POST /api/admin/system/restart`
|
||||
pub(crate) async fn restart(_auth: AdminAuth) -> Result<impl IntoResponse, AdminError> {
|
||||
if env_string("CPR_ENABLE_SELF_RESTART").as_deref() != Some("true") {
|
||||
return Err(AdminError::conflict(
|
||||
"自重启未启用,请设置 CPR_ENABLE_SELF_RESTART=true",
|
||||
));
|
||||
}
|
||||
|
||||
let config = SystemUpdateConfig::from_env();
|
||||
let plan = schedule_restart(&config)?;
|
||||
let message = plan.message;
|
||||
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
match plan.action {
|
||||
RestartAction::Exec(executable_path) => request_process_restart(executable_path),
|
||||
RestartAction::Shutdown => request_shutdown(),
|
||||
}
|
||||
});
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(serde_json::json!({
|
||||
"message": message,
|
||||
"operationId": operation_id("restart")
|
||||
})),
|
||||
))
|
||||
}
|
||||
|
||||
struct RestartPlan {
|
||||
message: &'static str,
|
||||
action: RestartAction,
|
||||
}
|
||||
|
||||
enum RestartAction {
|
||||
Exec(PathBuf),
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
fn schedule_restart(config: &SystemUpdateConfig) -> Result<RestartPlan, AdminError> {
|
||||
if config.deployment_mode == "docker" {
|
||||
return Ok(RestartPlan {
|
||||
message: "已安排进程内重启",
|
||||
action: RestartAction::Exec(config.executable_path()?),
|
||||
});
|
||||
}
|
||||
|
||||
spawn_replacement_process(config)?;
|
||||
Ok(RestartPlan {
|
||||
message: "已安排自重启",
|
||||
action: RestartAction::Shutdown,
|
||||
})
|
||||
}
|
||||
|
||||
fn spawn_replacement_process(config: &SystemUpdateConfig) -> Result<(), AdminError> {
|
||||
let executable_path = config.executable_path()?;
|
||||
let mut command = Command::new(executable_path);
|
||||
command
|
||||
.args(env::args_os().skip(1))
|
||||
.env(RESTART_DELAY_ENV, REPLACEMENT_START_DELAY_MS);
|
||||
command.spawn().map(|_| ()).map_err(internal_error_with(
|
||||
"Failed to schedule replacement process",
|
||||
))
|
||||
}
|
||||
|
||||
impl SystemUpdateConfig {
|
||||
fn from_env() -> Self {
|
||||
let update_repository = env_string("CPR_UPDATE_REPOSITORY");
|
||||
let deployment_mode =
|
||||
env_string("CPR_DEPLOYMENT_MODE").unwrap_or_else(|| "source".to_string());
|
||||
let executable_path = env_string("CPR_UPDATE_EXE_PATH")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| {
|
||||
(deployment_mode == "docker")
|
||||
.then(|| PathBuf::from("/app/bin").join(APP_BINARY_NAME))
|
||||
});
|
||||
let state_file = env_string("CPR_UPDATE_STATE_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("/app/data/update-state.json"));
|
||||
let update_lock_file = env_string("CPR_UPDATE_LOCK_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| state_file.with_extension("lock"));
|
||||
let update_temp_dir = env_string("CPR_UPDATE_TEMP_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| default_update_temp_dir(&state_file));
|
||||
Self {
|
||||
version: build_version(),
|
||||
git_sha: build_git_sha(),
|
||||
build_time: build_time(),
|
||||
deployment_mode,
|
||||
build_type: build_type(),
|
||||
update_channel: env_string("CPR_UPDATE_CHANNEL")
|
||||
.unwrap_or_else(|| "stable".to_string()),
|
||||
update_repository,
|
||||
github_api_base: env_string("CPR_GITHUB_API_BASE")
|
||||
.unwrap_or_else(|| GITHUB_API_BASE.to_string()),
|
||||
executable_path,
|
||||
web_dist_dir: env_string("CPR_WEB_DIST_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(DEFAULT_WEB_DIST_DIR)),
|
||||
update_state_file: state_file,
|
||||
update_lock_file,
|
||||
update_temp_dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn version_data(&self, update_info: &UpdateInfoData) -> VersionData {
|
||||
VersionData {
|
||||
version: self.version.clone(),
|
||||
git_sha: self.git_sha.clone(),
|
||||
build_time: self.build_time.clone(),
|
||||
deployment_mode: self.deployment_mode.clone(),
|
||||
deployment_mode_label: deployment_mode_label(&self.deployment_mode).to_string(),
|
||||
update_channel: self.update_channel.clone(),
|
||||
latest_version: update_info.latest_version.clone(),
|
||||
has_update: update_info.has_update,
|
||||
update_cached: update_info.cached,
|
||||
update_warning: update_info.warning.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_support_error(&self) -> Option<String> {
|
||||
if self.build_type != "release" {
|
||||
return Some("一键更新需要正式构建包".to_string());
|
||||
}
|
||||
if self.update_repository.is_none() {
|
||||
return Some("检查更新需要配置 CPR_UPDATE_REPOSITORY".to_string());
|
||||
}
|
||||
if let Err(error) = validate_github_api_base(&self.github_api_base) {
|
||||
return Some(error);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn release_cache_key(&self) -> String {
|
||||
format!(
|
||||
"{}|{}|{}|{}|{}",
|
||||
self.update_repository.as_deref().unwrap_or_default(),
|
||||
self.version,
|
||||
self.deployment_mode,
|
||||
self.build_type,
|
||||
self.update_channel,
|
||||
)
|
||||
}
|
||||
|
||||
fn executable_path(&self) -> Result<PathBuf, AdminError> {
|
||||
if let Some(path) = &self.executable_path {
|
||||
return Ok(path.clone());
|
||||
}
|
||||
env::current_exe()
|
||||
.and_then(fs::canonicalize)
|
||||
.map_err(internal_error_with("Failed to resolve executable"))
|
||||
}
|
||||
}
|
||||
|
||||
fn internal_error(context: &str, error: impl fmt::Display) -> AdminError {
|
||||
AdminError::internal(format!("{context}: {error}"))
|
||||
}
|
||||
|
||||
fn internal_error_with<E: fmt::Display>(context: &'static str) -> impl FnOnce(E) -> AdminError {
|
||||
move |error| internal_error(context, error)
|
||||
}
|
||||
|
||||
fn bad_request_with<E: fmt::Display>(context: &'static str) -> impl FnOnce(E) -> AdminError {
|
||||
move |error| AdminError::bad_request(format!("{context}: {error}"))
|
||||
}
|
||||
|
||||
fn bad_gateway_with<E: fmt::Display>(context: &'static str) -> impl FnOnce(E) -> AdminError {
|
||||
move |error| AdminError::bad_gateway(format!("{context}: {error}"))
|
||||
}
|
||||
|
||||
async fn perform_release_update(
|
||||
config: &SystemUpdateConfig,
|
||||
release: &GitHubRelease,
|
||||
version: &str,
|
||||
operation_id: &str,
|
||||
) -> Result<(), AdminError> {
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
"正在选择匹配当前平台的更新包",
|
||||
);
|
||||
let archive = select_release_archive(release, version)?;
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("asset"),
|
||||
format!(
|
||||
"已选择更新包 {} ({})",
|
||||
archive.name,
|
||||
format_bytes(archive.size)
|
||||
),
|
||||
);
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("verify"),
|
||||
"正在校验下载地址",
|
||||
);
|
||||
validate_download_url(&archive.browser_download_url, &config.github_api_base)?;
|
||||
if archive.size > MAX_DOWNLOAD_SIZE {
|
||||
return Err(AdminError::bad_request("Release archive is too large"));
|
||||
}
|
||||
let checksum = release
|
||||
.assets
|
||||
.iter()
|
||||
.find(|asset| asset.name == "checksums.txt")
|
||||
.ok_or_else(|| AdminError::bad_gateway("Release checksums.txt is required"))?;
|
||||
validate_download_url(&checksum.browser_download_url, &config.github_api_base)?;
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("prepare"),
|
||||
"正在创建临时更新目录",
|
||||
);
|
||||
fs::create_dir_all(&config.update_temp_dir)
|
||||
.map_err(internal_error_with("Failed to prepare update temp dir"))?;
|
||||
let temp_dir = fs::canonicalize(&config.update_temp_dir)
|
||||
.and_then(|dir| tempfile_dir_in(&dir, ".codex-proxy-rs-update-"))
|
||||
.map_err(internal_error_with("Failed to create update temp dir"))?;
|
||||
let archive_path = temp_dir.join(&archive.name);
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("download"),
|
||||
"开始下载更新包",
|
||||
);
|
||||
download_file(
|
||||
&archive.browser_download_url,
|
||||
&archive_path,
|
||||
MAX_DOWNLOAD_SIZE,
|
||||
Some(DownloadProgress {
|
||||
operation_id,
|
||||
total_size: archive.size,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Success,
|
||||
Some(operation_id),
|
||||
Some("download"),
|
||||
"更新包下载完成",
|
||||
);
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("checksum"),
|
||||
"正在校验 checksum",
|
||||
);
|
||||
verify_checksum(&archive_path, &archive.name, &checksum.browser_download_url).await?;
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Success,
|
||||
Some(operation_id),
|
||||
Some("checksum"),
|
||||
"checksum 校验通过",
|
||||
);
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("extract"),
|
||||
"正在解压更新包",
|
||||
);
|
||||
let extracted = extract_release_archive(&archive_path, &temp_dir)?;
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Success,
|
||||
Some(operation_id),
|
||||
Some("extract"),
|
||||
"更新包解压完成",
|
||||
);
|
||||
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Info,
|
||||
Some(operation_id),
|
||||
Some("replace"),
|
||||
"正在替换应用文件",
|
||||
);
|
||||
let exe_path = config.executable_path()?;
|
||||
replace_release_files(&exe_path, &config.web_dist_dir, extracted)?;
|
||||
emit_update_event(
|
||||
UpdateLogLevel::Success,
|
||||
Some(operation_id),
|
||||
Some("replace"),
|
||||
"应用文件替换完成",
|
||||
);
|
||||
let _ = fs::remove_dir_all(temp_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn tempfile_dir_in(parent: &Path, prefix: &str) -> io::Result<PathBuf> {
|
||||
for attempt in 0..100 {
|
||||
let path = parent.join(format!(
|
||||
"{prefix}{}-{attempt}",
|
||||
Utc::now().timestamp_nanos_opt().unwrap_or_default()
|
||||
));
|
||||
match fs::create_dir(&path) {
|
||||
Ok(()) => return Ok(path),
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::AlreadyExists,
|
||||
"failed to create unique temp dir",
|
||||
))
|
||||
}
|
||||
|
||||
fn default_update_temp_dir(state_file: &Path) -> PathBuf {
|
||||
state_file
|
||||
.parent()
|
||||
.map(|parent| parent.join("update-tmp"))
|
||||
.unwrap_or_else(|| env::temp_dir().join("codex-proxy-rs-update"))
|
||||
}
|
||||
|
||||
fn env_string(key: &str) -> Option<String> {
|
||||
env::var(key)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn runtime_build_info_override(key: &str) -> Option<String> {
|
||||
if cfg!(debug_assertions) {
|
||||
env_string(key)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn build_version() -> String {
|
||||
runtime_build_info_override("CPR_VERSION").unwrap_or_else(|| {
|
||||
option_env!("CPR_VERSION")
|
||||
.unwrap_or(env!("CARGO_PKG_VERSION"))
|
||||
.to_string()
|
||||
})
|
||||
}
|
||||
|
||||
fn build_git_sha() -> String {
|
||||
runtime_build_info_override("CPR_GIT_SHA")
|
||||
.unwrap_or_else(|| option_env!("CPR_GIT_SHA").unwrap_or("unknown").to_string())
|
||||
}
|
||||
|
||||
fn build_time() -> String {
|
||||
runtime_build_info_override("CPR_BUILD_TIME").unwrap_or_else(|| {
|
||||
option_env!("CPR_BUILD_TIME")
|
||||
.unwrap_or("unknown")
|
||||
.to_string()
|
||||
})
|
||||
}
|
||||
|
||||
fn build_type() -> String {
|
||||
runtime_build_info_override("CPR_BUILD_TYPE").unwrap_or_else(|| {
|
||||
option_env!("CPR_BUILD_TYPE")
|
||||
.unwrap_or("source")
|
||||
.to_string()
|
||||
})
|
||||
}
|
||||
|
||||
fn update_event_sender() -> &'static broadcast::Sender<SystemUpdateEvent> {
|
||||
UPDATE_EVENT_SENDER.get_or_init(|| {
|
||||
let (sender, _receiver) = broadcast::channel(256);
|
||||
sender
|
||||
})
|
||||
}
|
||||
|
||||
fn emit_update_event(
|
||||
level: UpdateLogLevel,
|
||||
operation_id: Option<&str>,
|
||||
step: Option<&str>,
|
||||
message: impl Into<String>,
|
||||
) {
|
||||
let now = Utc::now();
|
||||
let event = SystemUpdateEvent {
|
||||
id: format!(
|
||||
"update-log-{}",
|
||||
now.timestamp_nanos_opt()
|
||||
.unwrap_or_else(|| now.timestamp_millis())
|
||||
),
|
||||
operation_id: operation_id.map(ToString::to_string),
|
||||
level,
|
||||
step: step.map(ToString::to_string),
|
||||
message: message.into(),
|
||||
at: now.to_rfc3339(),
|
||||
};
|
||||
let _ = update_event_sender().send(event);
|
||||
}
|
||||
|
||||
fn format_bytes(bytes: u64) -> String {
|
||||
const KIB: u64 = 1024;
|
||||
const MIB: u64 = KIB * 1024;
|
||||
if bytes >= MIB {
|
||||
return format!("{:.1} MiB", bytes as f64 / MIB as f64);
|
||||
}
|
||||
if bytes >= KIB {
|
||||
return format!("{:.1} KiB", bytes as f64 / KIB as f64);
|
||||
}
|
||||
format!("{bytes} B")
|
||||
}
|
||||
|
||||
fn deployment_mode_label(value: &str) -> &str {
|
||||
match value {
|
||||
"docker" => "Docker",
|
||||
"source" => "源码部署",
|
||||
"binary" => "二进制部署",
|
||||
_ => value,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_type_label(value: &str) -> &str {
|
||||
match value {
|
||||
"release" => "正式构建",
|
||||
"source" => "源码构建",
|
||||
"dev" => "开发构建",
|
||||
_ => value,
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
//! 管理端通用更新请求解析。
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::admin::response::AdminError;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct EditableUpdateRequest {
|
||||
id: String,
|
||||
label: Option<Option<String>>,
|
||||
status: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) struct ParsedEditableUpdate {
|
||||
pub id: String,
|
||||
pub label: Option<Option<String>>,
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) struct EditableUpdateMessages<'a> {
|
||||
pub object_required: &'a str,
|
||||
pub invalid: &'a str,
|
||||
pub empty_update: &'a str,
|
||||
pub unknown_field_editable: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn parse_editable_update(
|
||||
payload: &Value,
|
||||
messages: EditableUpdateMessages<'_>,
|
||||
) -> Result<ParsedEditableUpdate, AdminError> {
|
||||
let object = payload
|
||||
.as_object()
|
||||
.ok_or_else(|| AdminError::bad_request(messages.object_required))?;
|
||||
if messages.unknown_field_editable {
|
||||
for field in object.keys() {
|
||||
if !matches!(field.as_str(), "id" | "label" | "status") {
|
||||
return Err(AdminError::bad_request(format!("{field} is not editable")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let update = serde_json::from_value::<EditableUpdateRequest>(payload.clone())
|
||||
.map_err(|_| AdminError::bad_request(messages.invalid))?;
|
||||
if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) {
|
||||
return Err(AdminError::bad_request(messages.invalid));
|
||||
}
|
||||
if update.label.is_none() && update.status.is_none() {
|
||||
return Err(AdminError::bad_request(messages.empty_update));
|
||||
}
|
||||
|
||||
Ok(ParsedEditableUpdate {
|
||||
id: update.id,
|
||||
label: update.label,
|
||||
status: update.status,
|
||||
})
|
||||
}
|
||||
|
||||
fn is_blank(value: &str) -> bool {
|
||||
value.trim().is_empty()
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
use super::*;
|
||||
|
||||
/// `GET /api/admin/accounts/export`
|
||||
pub async fn export_accounts(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<AccountExportQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let ids = account_export_ids(query.ids.as_deref());
|
||||
|
||||
if query.confirm.as_deref() != Some(ACCOUNT_EXPORT_CONFIRMATION) {
|
||||
return Err(AdminError::bad_request(
|
||||
"account export requires confirm=export_sensitive_accounts",
|
||||
));
|
||||
}
|
||||
|
||||
match state.services.admin_accounts.export(ids).await {
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(result),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
use super::*;
|
||||
|
||||
/// `POST /api/admin/accounts/import`
|
||||
pub async fn import_accounts(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<Value>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
match state.services.admin_accounts.import(payload).await {
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountImportData {
|
||||
imported: result.imported,
|
||||
skipped: result.skipped,
|
||||
source_format: result.source_format.to_string(),
|
||||
}),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use super::*;
|
||||
|
||||
/// `POST /api/admin/accounts/refresh`
|
||||
pub async fn refresh_account(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<AccountActionRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let account_id = payload.id;
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.refresh_account(&account_id)
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountRefreshData::from(result)),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts/health-check`
|
||||
pub async fn health_check_accounts(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<AccountHealthCheckRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let stagger_ms = payload.stagger_ms.unwrap_or(3_000);
|
||||
if !(500..=30_000).contains(&stagger_ms) && stagger_ms != 0 {
|
||||
return Err(AdminError::bad_request(
|
||||
"stagger_ms must be between 500 and 30000",
|
||||
));
|
||||
}
|
||||
if let Some(concurrency) = payload.concurrency
|
||||
&& !(1..=10).contains(&concurrency)
|
||||
{
|
||||
return Err(AdminError::bad_request(
|
||||
"concurrency must be between 1 and 10",
|
||||
));
|
||||
}
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.health_check_accounts(payload.ids, stagger_ms, payload.concurrency.unwrap_or(2))
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountHealthCheckData::from(result)),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts/delete`
|
||||
pub async fn batch_delete_accounts(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<BatchDeleteAccountsRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.batch_delete(payload.ids)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
for account_id in &result.deleted_ids {
|
||||
state
|
||||
.services
|
||||
.session_affinity
|
||||
.forget_account(account_id)
|
||||
.await;
|
||||
}
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BatchDeleteData {
|
||||
deleted: result.deleted,
|
||||
not_found: result.not_found,
|
||||
}),
|
||||
))
|
||||
}
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts/update`
|
||||
pub async fn update_account(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<Value>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let ParsedAccountUpdate { id, update } = parse_account_update(&payload)?;
|
||||
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.update_account(&id, update)
|
||||
.await
|
||||
{
|
||||
Ok(Some(account)) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountUpdateData::Account(Box::new(account.into()))),
|
||||
)),
|
||||
Ok(None) => Err(account_not_found()),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,578 @@
|
||||
//! 账号管理 HTTP 处理器。
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
body::Body,
|
||||
extract::{Query, State},
|
||||
http::{
|
||||
StatusCode,
|
||||
header::{CACHE_CONTROL, CONNECTION, CONTENT_TYPE},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{
|
||||
ADMIN_OK_CODE, ADMIN_OK_MESSAGE, AdminEnvelope, AdminError, AdminResponse,
|
||||
BatchDeleteData, EditableUpdateMessages, PageMeta, parse_editable_update,
|
||||
},
|
||||
session::AdminAuth,
|
||||
},
|
||||
fleet::{
|
||||
account::{AccountListSort, AccountSortField, AccountStatus},
|
||||
manage::{
|
||||
AccountHealthCheck, AccountManageError, AccountRefreshOutcome, AccountRefreshResult,
|
||||
AccountUpdate, ManagedAccount, OAuthExchangeInput,
|
||||
quota_view::{
|
||||
AccountQuotaData, AccountQuotaUsageWindow, AccountQuotaWindowLocalUsage, quota_data,
|
||||
},
|
||||
},
|
||||
refresh::token_refresh_status_eligible,
|
||||
},
|
||||
infra::{
|
||||
format::{
|
||||
format_billing_amount, format_percent, format_plain_number, format_tokens,
|
||||
nonnegative_i64_to_u64,
|
||||
},
|
||||
json::{SortDirection, clamp_limit, clamp_page, total_pages},
|
||||
time::{china_datetime, china_relative_time, china_rfc3339},
|
||||
},
|
||||
telemetry::{
|
||||
account_usage::query::AccountUsageRecord,
|
||||
billing,
|
||||
buckets::query::{ModelUsageWindow, UsageBucketWindow},
|
||||
},
|
||||
};
|
||||
|
||||
const ACCOUNT_STATS_PAGE_LIMIT: u32 = 200;
|
||||
const ACCOUNT_EXPORT_CONFIRMATION: &str = "export_sensitive_accounts";
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountsQuery {
|
||||
page: Option<u32>,
|
||||
page_size: Option<u32>,
|
||||
search: Option<String>,
|
||||
status: Option<String>,
|
||||
sort_by: Option<String>,
|
||||
sort_direction: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct CreateAccountRequest {
|
||||
token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct BatchDeleteAccountsRequest {
|
||||
ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountActionRequest {
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountIdQuery {
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountExportQuery {
|
||||
ids: Option<String>,
|
||||
confirm: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountTestQuery {
|
||||
id: String,
|
||||
model_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AccountOAuthExchangeRequest {
|
||||
session_id: String,
|
||||
callback_url: Option<String>,
|
||||
code: Option<String>,
|
||||
state: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountModelsData {
|
||||
models: Vec<AccountModelData>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountModelData {
|
||||
id: String,
|
||||
label: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountData {
|
||||
id: String,
|
||||
email: Option<String>,
|
||||
account_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
label: Option<String>,
|
||||
plan_type: Option<String>,
|
||||
has_refresh_token: bool,
|
||||
status: String,
|
||||
display_status: String,
|
||||
token_refreshing: bool,
|
||||
access_token_expires_at: Option<String>,
|
||||
access_token_expires_at_display: Option<String>,
|
||||
added_at: String,
|
||||
added_at_display: String,
|
||||
updated_at: String,
|
||||
updated_at_display: String,
|
||||
quota: AccountQuotaData,
|
||||
usage: AdminAccountUsageData,
|
||||
}
|
||||
|
||||
impl From<ManagedAccount> for AdminAccountData {
|
||||
fn from(a: ManagedAccount) -> Self {
|
||||
Self::from_parts(a, None, None, Vec::new(), false)
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminAccountData {
|
||||
fn from_parts(
|
||||
a: ManagedAccount,
|
||||
usage: Option<&AccountUsageRecord>,
|
||||
quota: Option<AccountQuotaData>,
|
||||
models: Vec<AdminAccountModelUsageData>,
|
||||
token_refreshing: bool,
|
||||
) -> Self {
|
||||
let access_token_expires_at = a.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let access_token_expires_at_display =
|
||||
a.access_token_expires_at.as_ref().map(china_datetime);
|
||||
let display_status = account_display_status(a.status, token_refreshing).to_string();
|
||||
Self {
|
||||
id: a.id,
|
||||
email: a.email,
|
||||
account_id: a.account_id,
|
||||
user_id: a.user_id,
|
||||
label: a.label,
|
||||
plan_type: a.plan_type,
|
||||
has_refresh_token: a.has_refresh_token,
|
||||
status: a.status.as_str().to_string(),
|
||||
display_status,
|
||||
token_refreshing,
|
||||
access_token_expires_at,
|
||||
access_token_expires_at_display,
|
||||
added_at: china_rfc3339(&a.added_at),
|
||||
added_at_display: china_datetime(&a.added_at),
|
||||
updated_at: china_rfc3339(&a.updated_at),
|
||||
updated_at_display: china_datetime(&a.updated_at),
|
||||
quota: quota.unwrap_or_default(),
|
||||
usage: AdminAccountUsageData::from_usage(usage, models),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountUsageData {
|
||||
request_count: i64,
|
||||
request_count_display: String,
|
||||
empty_response_count: i64,
|
||||
input_tokens: i64,
|
||||
input_tokens_display: String,
|
||||
output_tokens: i64,
|
||||
output_tokens_display: String,
|
||||
cached_tokens: i64,
|
||||
cached_tokens_display: String,
|
||||
reasoning_tokens: i64,
|
||||
total_tokens: i64,
|
||||
total_tokens_display: String,
|
||||
image_input_tokens: i64,
|
||||
image_output_tokens: i64,
|
||||
image_tokens_display: String,
|
||||
image_request_count: i64,
|
||||
image_request_failed_count: i64,
|
||||
created_tokens: i64,
|
||||
created_tokens_display: String,
|
||||
read_tokens: i64,
|
||||
read_tokens_display: String,
|
||||
last_used_at: Option<String>,
|
||||
last_used_at_display: String,
|
||||
models: Vec<AdminAccountModelUsageData>,
|
||||
}
|
||||
|
||||
impl AdminAccountUsageData {
|
||||
fn from_usage(
|
||||
usage: Option<&AccountUsageRecord>,
|
||||
models: Vec<AdminAccountModelUsageData>,
|
||||
) -> Self {
|
||||
let request_count = usage.map_or(0, |usage| usage.window_request_count);
|
||||
let empty_response_count = 0;
|
||||
let input_tokens = usage.map_or(0, |usage| usage.window_input_tokens);
|
||||
let output_tokens = usage.map_or(0, |usage| usage.window_output_tokens);
|
||||
let cached_tokens = usage.map_or(0, |usage| usage.window_cached_tokens);
|
||||
let reasoning_tokens = 0;
|
||||
let total_tokens = input_tokens.saturating_add(output_tokens);
|
||||
let image_input_tokens = 0;
|
||||
let image_output_tokens = 0;
|
||||
let image_request_count = 0;
|
||||
let image_request_failed_count = 0;
|
||||
let created_tokens = input_tokens.saturating_sub(cached_tokens);
|
||||
let read_tokens = cached_tokens;
|
||||
let last_used_at = usage.and_then(|usage| usage.last_used_at);
|
||||
|
||||
Self {
|
||||
request_count,
|
||||
request_count_display: format_plain_number(nonnegative_i64_to_u64(request_count)),
|
||||
empty_response_count,
|
||||
input_tokens,
|
||||
input_tokens_display: format_tokens(nonnegative_i64_to_u64(input_tokens)),
|
||||
output_tokens,
|
||||
output_tokens_display: format_tokens(nonnegative_i64_to_u64(output_tokens)),
|
||||
cached_tokens,
|
||||
cached_tokens_display: format_tokens(nonnegative_i64_to_u64(cached_tokens)),
|
||||
reasoning_tokens,
|
||||
total_tokens,
|
||||
total_tokens_display: format_tokens(nonnegative_i64_to_u64(total_tokens)),
|
||||
image_input_tokens,
|
||||
image_output_tokens,
|
||||
image_tokens_display: format_tokens(nonnegative_i64_to_u64(
|
||||
image_input_tokens + image_output_tokens,
|
||||
)),
|
||||
image_request_count,
|
||||
image_request_failed_count,
|
||||
created_tokens,
|
||||
created_tokens_display: format_tokens(nonnegative_i64_to_u64(created_tokens)),
|
||||
read_tokens,
|
||||
read_tokens_display: format_tokens(nonnegative_i64_to_u64(read_tokens)),
|
||||
last_used_at: last_used_at.map(|value| china_rfc3339(&value)),
|
||||
last_used_at_display: china_relative_time(last_used_at, Utc::now()),
|
||||
models,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountModelUsageData {
|
||||
model: String,
|
||||
request_count: u64,
|
||||
request_count_display: String,
|
||||
success_rate: f64,
|
||||
success_rate_display: String,
|
||||
input_tokens: u64,
|
||||
input_tokens_display: String,
|
||||
output_tokens: u64,
|
||||
output_tokens_display: String,
|
||||
cached_tokens: u64,
|
||||
cached_tokens_display: String,
|
||||
total_tokens: u64,
|
||||
total_tokens_display: String,
|
||||
billing_amount_usd: f64,
|
||||
billing_amount_usd_display: String,
|
||||
last_used_at: Option<String>,
|
||||
last_used_at_display: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct AccountListStats {
|
||||
usage_by_account: HashMap<String, AccountUsageRecord>,
|
||||
quota_by_account: HashMap<String, AccountQuotaData>,
|
||||
models_by_account: HashMap<String, Vec<AdminAccountModelUsageData>>,
|
||||
refreshing_account_ids: HashSet<String>,
|
||||
}
|
||||
|
||||
struct AccountModelUsageRecord {
|
||||
account_id: String,
|
||||
model: String,
|
||||
request_count: i64,
|
||||
error_count: i64,
|
||||
input_tokens: i64,
|
||||
output_tokens: i64,
|
||||
cached_tokens: i64,
|
||||
billing_amount_usd: f64,
|
||||
last_used_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct AccountQuotaWindowSelection {
|
||||
account_id: String,
|
||||
window: AccountQuotaUsageWindow,
|
||||
}
|
||||
|
||||
impl AccountListStats {
|
||||
fn data_for(&self, account: ManagedAccount) -> AdminAccountData {
|
||||
let account_id = account.id.clone();
|
||||
let token_refreshing = token_refresh_status_eligible(account.status)
|
||||
&& self.refreshing_account_ids.contains(&account_id);
|
||||
AdminAccountData::from_parts(
|
||||
account,
|
||||
self.usage_by_account.get(&account_id),
|
||||
self.quota_by_account.get(&account_id).cloned(),
|
||||
self.models_by_account
|
||||
.get(&account_id)
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
token_refreshing,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountSummaryData {
|
||||
total: u64,
|
||||
active: u64,
|
||||
quota_exhausted: u64,
|
||||
attention: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountPageData {
|
||||
items: Vec<AdminAccountData>,
|
||||
page: PageMeta,
|
||||
summary: AdminAccountSummaryData,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountPageEnvelope {
|
||||
code: u32,
|
||||
message: String,
|
||||
data: AdminAccountPageData,
|
||||
}
|
||||
|
||||
impl AdminAccountPageEnvelope {
|
||||
fn ok(
|
||||
page: crate::infra::json::NumberedPage<AdminAccountData>,
|
||||
summary: AdminAccountSummaryData,
|
||||
) -> Self {
|
||||
Self {
|
||||
code: ADMIN_OK_CODE,
|
||||
message: ADMIN_OK_MESSAGE.into(),
|
||||
data: AdminAccountPageData {
|
||||
page: PageMeta {
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
total: page.total,
|
||||
total_pages: total_pages(page.total, page.page_size),
|
||||
},
|
||||
items: page.items,
|
||||
summary,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum AccountUpdateData {
|
||||
Account(Box<AdminAccountData>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountRefreshData {
|
||||
id: String,
|
||||
email: Option<String>,
|
||||
previous_status: String,
|
||||
result: &'static str,
|
||||
error: Option<String>,
|
||||
duration_ms: i64,
|
||||
}
|
||||
|
||||
impl From<AccountRefreshResult> for AccountRefreshData {
|
||||
fn from(result: AccountRefreshResult) -> Self {
|
||||
Self {
|
||||
id: result.id,
|
||||
email: result.email,
|
||||
previous_status: result.previous_status.as_str().to_string(),
|
||||
result: account_refresh_outcome_str(result.outcome),
|
||||
error: result.error,
|
||||
duration_ms: result.duration_ms,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminAccountQuotaResponseData {
|
||||
quota: Value,
|
||||
raw: Value,
|
||||
quota_data: AccountQuotaData,
|
||||
plan_type: Option<String>,
|
||||
account: AdminAccountData,
|
||||
}
|
||||
|
||||
impl AdminAccountQuotaResponseData {
|
||||
fn from_account(
|
||||
quota: Value,
|
||||
raw: Value,
|
||||
quota_data: AccountQuotaData,
|
||||
account: AdminAccountData,
|
||||
) -> Self {
|
||||
let plan_type = account.plan_type.clone();
|
||||
Self {
|
||||
quota,
|
||||
raw,
|
||||
quota_data,
|
||||
plan_type,
|
||||
account,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct AccountHealthCheckRequest {
|
||||
ids: Option<Vec<String>>,
|
||||
stagger_ms: Option<u64>,
|
||||
concurrency: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountHealthCheckSummaryData {
|
||||
total: usize,
|
||||
alive: usize,
|
||||
dead: usize,
|
||||
skipped: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountHealthCheckData {
|
||||
summary: AccountHealthCheckSummaryData,
|
||||
results: Vec<AccountRefreshData>,
|
||||
}
|
||||
|
||||
impl From<AccountHealthCheck> for AccountHealthCheckData {
|
||||
fn from(result: AccountHealthCheck) -> Self {
|
||||
let summary = AccountHealthCheckSummaryData {
|
||||
total: result.results.len(),
|
||||
alive: result.alive(),
|
||||
dead: result.dead(),
|
||||
skipped: result.skipped(),
|
||||
};
|
||||
Self {
|
||||
summary,
|
||||
results: result
|
||||
.results
|
||||
.into_iter()
|
||||
.map(AccountRefreshData::from)
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountImportData {
|
||||
imported: u32,
|
||||
skipped: u32,
|
||||
source_format: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountOAuthAuthorizeData {
|
||||
session_id: String,
|
||||
auth_url: String,
|
||||
expires_at: String,
|
||||
expires_at_display: String,
|
||||
}
|
||||
|
||||
/// `GET /api/admin/accounts`
|
||||
pub(crate) async fn accounts(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(params): Query<AccountsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let page = clamp_page(params.page.unwrap_or(1));
|
||||
let page_size = clamp_limit(params.page_size.unwrap_or(50));
|
||||
let status = account_status_filter(params.status)?;
|
||||
let sort = account_list_sort(params.sort_by, params.sort_direction)?;
|
||||
let quota_by_account = quota_snapshots_by_account(&state).await;
|
||||
let summary = account_summary_data(&state).await;
|
||||
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.list_page(page, page_size, params.search, status, sort)
|
||||
.await
|
||||
{
|
||||
Ok(page) => {
|
||||
let stats = account_list_stats(&state, &page.items, "a_by_account).await?;
|
||||
let page = crate::infra::json::NumberedPage {
|
||||
items: page
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| stats.data_for(item))
|
||||
.collect(),
|
||||
total: page.total,
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
};
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminAccountPageEnvelope::ok(page, summary),
|
||||
))
|
||||
}
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts`
|
||||
pub(crate) async fn create_account(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<CreateAccountRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.create(payload.token, payload.refresh_token)
|
||||
.await
|
||||
{
|
||||
Ok(account) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminAccountData::from(account)),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
mod export_routes;
|
||||
mod import_routes;
|
||||
mod lifecycle_routes;
|
||||
mod oauth_routes;
|
||||
mod probe_routes;
|
||||
mod query;
|
||||
mod quota_routes;
|
||||
|
||||
pub(crate) use export_routes::*;
|
||||
pub(crate) use import_routes::*;
|
||||
pub(crate) use lifecycle_routes::*;
|
||||
pub(crate) use oauth_routes::*;
|
||||
pub(crate) use probe_routes::*;
|
||||
use query::*;
|
||||
pub(crate) use quota_routes::*;
|
||||
@@ -0,0 +1,49 @@
|
||||
use super::*;
|
||||
|
||||
/// `POST /api/admin/accounts/oauth/authorize`
|
||||
pub async fn oauth_authorize_account(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
match state.services.admin_accounts.oauth_authorize().await {
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountOAuthAuthorizeData {
|
||||
session_id: result.session_id,
|
||||
auth_url: result.auth_url,
|
||||
expires_at: china_rfc3339(&result.expires_at),
|
||||
expires_at_display: china_datetime(&result.expires_at),
|
||||
}),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts/oauth/exchange`
|
||||
pub async fn oauth_exchange_account(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Json(payload): Json<AccountOAuthExchangeRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.oauth_exchange(OAuthExchangeInput {
|
||||
session_id: payload.session_id,
|
||||
callback_url: payload.callback_url,
|
||||
code: payload.code,
|
||||
state: payload.state,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountImportData {
|
||||
imported: result.imported,
|
||||
skipped: result.skipped,
|
||||
source_format: result.source_format.to_string(),
|
||||
}),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use super::*;
|
||||
|
||||
/// `GET /api/admin/accounts/test?id=...&modelId=...`
|
||||
pub async fn test_account_connection(
|
||||
State(state): State<AppState>,
|
||||
Query(query): Query<AccountTestQuery>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<Response, AdminError> {
|
||||
let account_id = query.id;
|
||||
|
||||
let model = query
|
||||
.model_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
.ok_or_else(|| AdminError::bad_request("Model is required"))?;
|
||||
let stream = state
|
||||
.services
|
||||
.admin_accounts
|
||||
.test_connection_stream(&account_id, model)
|
||||
.await
|
||||
.map_err(|error| account_error(&error))?;
|
||||
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(CONTENT_TYPE, "text/event-stream")
|
||||
.header(CACHE_CONTROL, "no-cache")
|
||||
.header(CONNECTION, "keep-alive")
|
||||
.header("x-accel-buffering", "no")
|
||||
.body(Body::from_stream(stream))
|
||||
.map_err(|_| AdminError::internal("Failed to build account test stream"))
|
||||
}
|
||||
|
||||
/// `GET /api/admin/accounts/models?id=...`
|
||||
pub async fn account_models(
|
||||
State(state): State<AppState>,
|
||||
Query(query): Query<AccountIdQuery>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let account_id = query.id;
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.account_models(&account_id)
|
||||
.await
|
||||
{
|
||||
Ok(models) => Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountModelsData {
|
||||
models: models
|
||||
.into_iter()
|
||||
.map(|model| AccountModelData {
|
||||
id: model.id,
|
||||
label: model.label,
|
||||
})
|
||||
.collect(),
|
||||
}),
|
||||
)),
|
||||
Err(error) => Err(account_error(&error)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn account_list_stats(
|
||||
state: &AppState,
|
||||
accounts: &[ManagedAccount],
|
||||
quota_by_account: &HashMap<String, AccountQuotaData>,
|
||||
) -> Result<AccountListStats, AdminError> {
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let usage_records = state
|
||||
.services
|
||||
.usage
|
||||
.list_by_account_ids(&account_ids)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut quota_by_account = quota_by_account.clone();
|
||||
let all_quota_windows = quota_usage_windows_by_account(accounts, "a_by_account);
|
||||
let quota_window_usage_stats =
|
||||
quota_window_local_usage_by_account(state, &all_quota_windows).await;
|
||||
apply_quota_window_local_usage(accounts, &mut quota_by_account, "a_window_usage_stats);
|
||||
let selected_quota_windows = selected_quota_windows_by_account(accounts, "a_by_account);
|
||||
let usage_records = usage_records
|
||||
.into_iter()
|
||||
.map(|usage| {
|
||||
apply_selected_quota_window_usage(
|
||||
usage,
|
||||
&selected_quota_windows,
|
||||
"a_window_usage_stats,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let models_by_account = list_current_window_model_usage(state, &usage_records).await;
|
||||
let refreshing_account_ids = state
|
||||
.services
|
||||
.token_refresh
|
||||
.refreshing_account_ids(&account_ids, Utc::now())
|
||||
.await
|
||||
.map_err(|error| AdminError::internal(error.to_string()))?;
|
||||
|
||||
Ok(AccountListStats {
|
||||
usage_by_account: usage_records
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect(),
|
||||
quota_by_account,
|
||||
models_by_account,
|
||||
refreshing_account_ids,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn account_display_status(
|
||||
status: AccountStatus,
|
||||
token_refreshing: bool,
|
||||
) -> &'static str {
|
||||
if token_refreshing {
|
||||
"refreshing"
|
||||
} else {
|
||||
status.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn selected_quota_windows_by_account(
|
||||
accounts: &[ManagedAccount],
|
||||
quota_by_account: &HashMap<String, AccountQuotaData>,
|
||||
) -> HashMap<String, AccountQuotaUsageWindow> {
|
||||
accounts
|
||||
.iter()
|
||||
.filter_map(|account| {
|
||||
quota_by_account
|
||||
.get(&account.id)
|
||||
.and_then(selected_quota_window)
|
||||
.map(|window| (account.id.clone(), window))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn selected_quota_window(quota: &AccountQuotaData) -> Option<AccountQuotaUsageWindow> {
|
||||
quota.usage_windows().into_iter().max_by(|left, right| {
|
||||
left.duration_seconds()
|
||||
.cmp(&right.duration_seconds())
|
||||
.then_with(|| left.end.cmp(&right.end))
|
||||
.then_with(|| left.key.cmp(&right.key))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn quota_usage_windows_by_account(
|
||||
accounts: &[ManagedAccount],
|
||||
quota_by_account: &HashMap<String, AccountQuotaData>,
|
||||
) -> Vec<AccountQuotaWindowSelection> {
|
||||
accounts
|
||||
.iter()
|
||||
.filter_map(|account| {
|
||||
quota_by_account
|
||||
.get(&account.id)
|
||||
.map(|quota| (account.id.as_str(), quota))
|
||||
})
|
||||
.flat_map(|(account_id, quota)| {
|
||||
quota
|
||||
.usage_windows()
|
||||
.into_iter()
|
||||
.map(move |window| AccountQuotaWindowSelection {
|
||||
account_id: account_id.to_string(),
|
||||
window,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) async fn apply_account_quota_window_local_usage(
|
||||
state: &AppState,
|
||||
account_id: &str,
|
||||
quota: &mut AccountQuotaData,
|
||||
) {
|
||||
let windows = quota
|
||||
.usage_windows()
|
||||
.into_iter()
|
||||
.map(|window| AccountQuotaWindowSelection {
|
||||
account_id: account_id.to_string(),
|
||||
window,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let usage_by_account = quota_window_local_usage_by_account(state, &windows).await;
|
||||
let empty_usage = HashMap::new();
|
||||
quota.apply_local_usage(usage_by_account.get(account_id).unwrap_or(&empty_usage));
|
||||
}
|
||||
|
||||
pub(super) fn apply_quota_window_local_usage(
|
||||
accounts: &[ManagedAccount],
|
||||
quota_by_account: &mut HashMap<String, AccountQuotaData>,
|
||||
usage_by_account: &HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>>,
|
||||
) {
|
||||
for account in accounts {
|
||||
if let Some(quota) = quota_by_account.get_mut(&account.id) {
|
||||
let empty_usage = HashMap::new();
|
||||
quota.apply_local_usage(usage_by_account.get(&account.id).unwrap_or(&empty_usage));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn quota_window_local_usage_by_account(
|
||||
state: &AppState,
|
||||
windows: &[AccountQuotaWindowSelection],
|
||||
) -> HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>> {
|
||||
let queries = windows
|
||||
.iter()
|
||||
.map(|window| UsageBucketWindow {
|
||||
account_id: window.account_id.clone(),
|
||||
key: window.window.key.clone(),
|
||||
start: window.window.start,
|
||||
end: window.window.end,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
state
|
||||
.services
|
||||
.usage
|
||||
.usage_by_windows(&queries)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|(account_id, usage_by_window)| {
|
||||
let usage_by_window = usage_by_window
|
||||
.into_iter()
|
||||
.map(|(key, usage)| {
|
||||
(
|
||||
key,
|
||||
AccountQuotaWindowLocalUsage {
|
||||
request_count: usage.request_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
(account_id, usage_by_window)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn apply_selected_quota_window_usage(
|
||||
mut usage: AccountUsageRecord,
|
||||
selected_quota_windows: &HashMap<String, AccountQuotaUsageWindow>,
|
||||
quota_window_usage_stats: &HashMap<String, HashMap<String, AccountQuotaWindowLocalUsage>>,
|
||||
) -> AccountUsageRecord {
|
||||
let Some(window) = selected_quota_windows.get(&usage.account_id) else {
|
||||
return usage;
|
||||
};
|
||||
let stats = quota_window_usage_stats
|
||||
.get(&usage.account_id)
|
||||
.and_then(|usage_by_window| usage_by_window.get(&window.key))
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
usage.window_request_count = stats.request_count;
|
||||
usage.window_input_tokens = stats.input_tokens;
|
||||
usage.window_output_tokens = stats.output_tokens;
|
||||
usage.window_cached_tokens = stats.cached_tokens;
|
||||
usage.window_started_at = Some(window.start);
|
||||
usage.window_reset_at = Some(window.end);
|
||||
usage.limit_window_seconds = Some(window.window_seconds);
|
||||
usage
|
||||
}
|
||||
|
||||
pub(super) async fn account_summary_data(state: &AppState) -> AdminAccountSummaryData {
|
||||
let accounts = list_all_account_metadata(state).await;
|
||||
let total = accounts.len() as u64;
|
||||
let active = accounts
|
||||
.iter()
|
||||
.filter(|account| account.status == AccountStatus::Active)
|
||||
.count() as u64;
|
||||
let quota_exhausted = accounts
|
||||
.iter()
|
||||
.filter(|account| account.status == AccountStatus::QuotaExhausted)
|
||||
.count() as u64;
|
||||
let attention = accounts
|
||||
.iter()
|
||||
.filter(|account| account_summary_needs_attention(account.status))
|
||||
.count() as u64;
|
||||
|
||||
AdminAccountSummaryData {
|
||||
total,
|
||||
active,
|
||||
quota_exhausted,
|
||||
attention,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn quota_snapshots_by_account(
|
||||
state: &AppState,
|
||||
) -> HashMap<String, AccountQuotaData> {
|
||||
state
|
||||
.services
|
||||
.admin_accounts
|
||||
.quota_snapshots()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|snapshot| {
|
||||
(
|
||||
snapshot.account_id,
|
||||
quota_data(&snapshot.quota_json, snapshot.quota_fetched_at),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) async fn list_all_account_metadata(state: &AppState) -> Vec<ManagedAccount> {
|
||||
let mut page = 1;
|
||||
let mut accounts = Vec::new();
|
||||
loop {
|
||||
let Ok(result) = state
|
||||
.services
|
||||
.admin_accounts
|
||||
.list_page(page, ACCOUNT_STATS_PAGE_LIMIT, None, None, None)
|
||||
.await
|
||||
else {
|
||||
return Vec::new();
|
||||
};
|
||||
let total = result.total;
|
||||
accounts.extend(result.items);
|
||||
if accounts.len() as u64 >= total || total == 0 {
|
||||
return accounts;
|
||||
}
|
||||
page = page.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn account_summary_needs_attention(status: AccountStatus) -> bool {
|
||||
matches!(
|
||||
status,
|
||||
AccountStatus::Expired | AccountStatus::Disabled | AccountStatus::Banned
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn account_status_filter(
|
||||
status: Option<String>,
|
||||
) -> Result<Option<AccountStatus>, AdminError> {
|
||||
let Some(status) = status.map(|value| value.trim().to_string()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if status.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
AccountStatus::parse(&status)
|
||||
.map(Some)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid account status"))
|
||||
}
|
||||
|
||||
pub(super) fn account_list_sort(
|
||||
sort_by: Option<String>,
|
||||
sort_direction: Option<String>,
|
||||
) -> Result<Option<AccountListSort>, AdminError> {
|
||||
let (sort_by, sort_direction) = match (sort_by, sort_direction) {
|
||||
(None, None) => return Ok(None),
|
||||
(Some(sort_by), Some(sort_direction)) => (sort_by, sort_direction),
|
||||
_ => {
|
||||
return Err(AdminError::bad_request(
|
||||
"Account sort field and direction must be provided together",
|
||||
));
|
||||
}
|
||||
};
|
||||
let field = match sort_by.trim() {
|
||||
"email" => AccountSortField::Email,
|
||||
"status" => AccountSortField::Status,
|
||||
"planType" => AccountSortField::PlanType,
|
||||
"usage" => AccountSortField::Usage,
|
||||
"lastUsedAt" => AccountSortField::LastUsedAt,
|
||||
"expiresAt" => AccountSortField::ExpiresAt,
|
||||
_ => return Err(AdminError::bad_request("Invalid account sort field")),
|
||||
};
|
||||
let direction = SortDirection::parse(&sort_direction)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid account sort direction"))?;
|
||||
Ok(Some(AccountListSort { field, direction }))
|
||||
}
|
||||
|
||||
pub(super) async fn list_current_window_model_usage(
|
||||
state: &AppState,
|
||||
usage_records: &[AccountUsageRecord],
|
||||
) -> HashMap<String, Vec<AdminAccountModelUsageData>> {
|
||||
let now = Utc::now();
|
||||
let windows = usage_records
|
||||
.iter()
|
||||
.filter_map(|usage| {
|
||||
current_usage_window(usage, now)
|
||||
.map(|(start, end)| (usage.account_id.clone(), start, end))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if windows.is_empty() {
|
||||
return HashMap::new();
|
||||
}
|
||||
|
||||
let windows = windows
|
||||
.into_iter()
|
||||
.map(|(account_id, start, end)| ModelUsageWindow {
|
||||
account_id,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let rows = state
|
||||
.services
|
||||
.usage
|
||||
.model_usage_by_windows(&windows)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut records_by_model = HashMap::<(String, String), AccountModelUsageRecord>::new();
|
||||
for row in rows {
|
||||
let account_id = row.account_id;
|
||||
let model = row.model;
|
||||
let request_count = row.request_count;
|
||||
let error_count = row.error_count;
|
||||
let input_tokens = row.input_tokens;
|
||||
let output_tokens = row.output_tokens;
|
||||
let cached_tokens = row.cached_tokens;
|
||||
let billing_amount_usd = billing::calculate_billing_amount(
|
||||
nonnegative_i64_to_u64(input_tokens),
|
||||
nonnegative_i64_to_u64(output_tokens),
|
||||
nonnegative_i64_to_u64(cached_tokens),
|
||||
&model,
|
||||
row.service_tier.as_deref(),
|
||||
);
|
||||
let last_used_at = row.last_used_at;
|
||||
|
||||
let record = records_by_model
|
||||
.entry((account_id.clone(), model.clone()))
|
||||
.or_insert_with(|| AccountModelUsageRecord {
|
||||
account_id,
|
||||
model,
|
||||
request_count: 0,
|
||||
error_count: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cached_tokens: 0,
|
||||
billing_amount_usd: 0.0,
|
||||
last_used_at: None,
|
||||
});
|
||||
record.request_count += request_count;
|
||||
record.error_count += error_count;
|
||||
record.input_tokens += input_tokens;
|
||||
record.output_tokens += output_tokens;
|
||||
record.cached_tokens += cached_tokens;
|
||||
record.billing_amount_usd += billing_amount_usd;
|
||||
record.last_used_at = record.last_used_at.max(last_used_at);
|
||||
}
|
||||
let records = records_by_model.into_values().collect::<Vec<_>>();
|
||||
|
||||
models_by_account(records)
|
||||
}
|
||||
|
||||
pub(super) fn current_usage_window(
|
||||
usage: &AccountUsageRecord,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<(DateTime<Utc>, DateTime<Utc>)> {
|
||||
let start = usage.window_started_at?;
|
||||
let end = usage.window_reset_at.unwrap_or(now);
|
||||
(start <= end).then_some((start, end))
|
||||
}
|
||||
|
||||
pub(super) fn models_by_account(
|
||||
records: Vec<AccountModelUsageRecord>,
|
||||
) -> HashMap<String, Vec<AdminAccountModelUsageData>> {
|
||||
let mut by_account = HashMap::<String, Vec<AccountModelUsageRecord>>::new();
|
||||
for record in records {
|
||||
by_account
|
||||
.entry(record.account_id.clone())
|
||||
.or_default()
|
||||
.push(record);
|
||||
}
|
||||
|
||||
by_account
|
||||
.into_iter()
|
||||
.map(|(account_id, mut records)| {
|
||||
records.sort_by(|a, b| {
|
||||
b.request_count
|
||||
.cmp(&a.request_count)
|
||||
.then_with(|| b.last_used_at.cmp(&a.last_used_at))
|
||||
.then_with(|| a.model.cmp(&b.model))
|
||||
});
|
||||
(
|
||||
account_id,
|
||||
records.into_iter().map(model_usage_data).collect(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn model_usage_data(usage: AccountModelUsageRecord) -> AdminAccountModelUsageData {
|
||||
let request_count = nonnegative_i64_to_u64(usage.request_count);
|
||||
let error_count = nonnegative_i64_to_u64(usage.error_count);
|
||||
let input_tokens = nonnegative_i64_to_u64(usage.input_tokens);
|
||||
let output_tokens = nonnegative_i64_to_u64(usage.output_tokens);
|
||||
let cached_tokens = nonnegative_i64_to_u64(usage.cached_tokens);
|
||||
let total_tokens = input_tokens + output_tokens;
|
||||
let success_rate = if request_count > 0 {
|
||||
((request_count.saturating_sub(error_count)) as f64 / request_count as f64 * 1000.0).round()
|
||||
/ 10.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let billing_amount_usd = usage.billing_amount_usd;
|
||||
|
||||
AdminAccountModelUsageData {
|
||||
model: usage.model,
|
||||
request_count,
|
||||
request_count_display: format_plain_number(request_count),
|
||||
success_rate,
|
||||
success_rate_display: format_percent(success_rate),
|
||||
input_tokens,
|
||||
input_tokens_display: format_tokens(input_tokens),
|
||||
output_tokens,
|
||||
output_tokens_display: format_tokens(output_tokens),
|
||||
cached_tokens,
|
||||
cached_tokens_display: format_tokens(cached_tokens),
|
||||
total_tokens,
|
||||
total_tokens_display: format_tokens(total_tokens),
|
||||
billing_amount_usd,
|
||||
billing_amount_usd_display: format_billing_amount(billing_amount_usd),
|
||||
last_used_at: usage.last_used_at.map(|value| china_rfc3339(&value)),
|
||||
last_used_at_display: china_relative_time(usage.last_used_at, Utc::now()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ParsedAccountUpdate {
|
||||
pub(super) id: String,
|
||||
pub(super) update: AccountUpdate,
|
||||
}
|
||||
|
||||
pub(super) fn parse_account_update(payload: &Value) -> Result<ParsedAccountUpdate, AdminError> {
|
||||
let payload = parse_editable_update(
|
||||
payload,
|
||||
EditableUpdateMessages {
|
||||
object_required: "Account update request must be an object",
|
||||
invalid: "Invalid account update request",
|
||||
empty_update: "Account update request must include editable fields",
|
||||
unknown_field_editable: true,
|
||||
},
|
||||
)?;
|
||||
let update = AccountUpdate {
|
||||
label: payload.label.map(|label| {
|
||||
label.and_then(|value| {
|
||||
let value = value.trim();
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
})
|
||||
}),
|
||||
status: payload.status,
|
||||
};
|
||||
if !update.any() {
|
||||
return Err(AdminError::bad_request(
|
||||
"Account update request must include editable fields",
|
||||
));
|
||||
}
|
||||
Ok(ParsedAccountUpdate {
|
||||
id: payload.id,
|
||||
update,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn account_error(error: &AccountManageError) -> AdminError {
|
||||
match error {
|
||||
AccountManageError::InvalidStatus(_)
|
||||
| AccountManageError::LabelTooLong
|
||||
| AccountManageError::EmptyIds
|
||||
| AccountManageError::NoImportableAccounts
|
||||
| AccountManageError::NoModels
|
||||
| AccountManageError::InvalidAccessTokenExpiresAt
|
||||
| AccountManageError::TokenRequired
|
||||
| AccountManageError::InvalidToken(_)
|
||||
| AccountManageError::RefreshTokenExchange(_)
|
||||
| AccountManageError::OAuthSessionInvalid
|
||||
| AccountManageError::OAuthCallbackInvalid
|
||||
| AccountManageError::OAuthStateMismatch
|
||||
| AccountManageError::NoValidCookies => AdminError::bad_request(error.to_string()),
|
||||
AccountManageError::OAuthCodeExchange(_) => AdminError::bad_gateway(error.to_string()),
|
||||
AccountManageError::NotFound => account_not_found(),
|
||||
AccountManageError::Inactive(_) => AdminError::conflict(error.to_string()),
|
||||
_ => AdminError::internal(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn account_refresh_outcome_str(outcome: AccountRefreshOutcome) -> &'static str {
|
||||
match outcome {
|
||||
AccountRefreshOutcome::Alive => "alive",
|
||||
AccountRefreshOutcome::Dead => "dead",
|
||||
AccountRefreshOutcome::Skipped => "skipped",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn account_not_found() -> AdminError {
|
||||
AdminError::not_found("Account not found")
|
||||
}
|
||||
|
||||
pub(super) fn account_export_ids(value: Option<&str>) -> Vec<String> {
|
||||
value
|
||||
.into_iter()
|
||||
.flat_map(|ids| ids.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|id| !id.is_empty())
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
use super::*;
|
||||
|
||||
/// `GET /api/admin/accounts/quota`
|
||||
pub async fn account_quota(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<AccountIdQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let account_id = query.id;
|
||||
match state
|
||||
.services
|
||||
.admin_accounts
|
||||
.account_quota(&account_id)
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
let quota = data.get("quota").cloned().unwrap_or(Value::Null);
|
||||
let raw = data.get("raw").cloned().unwrap_or(Value::Null);
|
||||
let quota_json = quota.to_string();
|
||||
let mut quota_data = quota_data("a_json, Some(Utc::now()));
|
||||
apply_account_quota_window_local_usage(&state, &account_id, &mut quota_data).await;
|
||||
let account =
|
||||
account_data_for_quota_refresh(&state, &account_id, quota_data.clone()).await?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminAccountQuotaResponseData::from_account(
|
||||
quota, raw, quota_data, account,
|
||||
)),
|
||||
))
|
||||
}
|
||||
Err(AccountManageError::NotFound) => Err(account_not_found()),
|
||||
Err(AccountManageError::Inactive(status)) => Err(AdminError::conflict(format!(
|
||||
"Account is {}, cannot query quota",
|
||||
status.as_str()
|
||||
))),
|
||||
Err(AccountManageError::FetchQuota(msg)) => Err(AdminError::bad_gateway(format!(
|
||||
"Failed to fetch quota from Codex API: {msg}"
|
||||
))),
|
||||
Err(e) => Err(account_error(&e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn account_data_for_quota_refresh(
|
||||
state: &AppState,
|
||||
account_id: &str,
|
||||
quota_data: AccountQuotaData,
|
||||
) -> Result<AdminAccountData, AdminError> {
|
||||
let Some(account) = state
|
||||
.services
|
||||
.admin_accounts
|
||||
.get(account_id)
|
||||
.await
|
||||
.map_err(|error| account_error(&error))?
|
||||
else {
|
||||
return Err(account_not_found());
|
||||
};
|
||||
let quota_by_account = HashMap::from([(account_id.to_string(), quota_data)]);
|
||||
let stats =
|
||||
account_list_stats(state, std::slice::from_ref(&account), "a_by_account).await?;
|
||||
Ok(stats.data_for(account))
|
||||
}
|
||||
@@ -1,41 +1,28 @@
|
||||
//! 管理端会话处理器。
|
||||
//! 管理端认证路由。
|
||||
|
||||
use axum::{
|
||||
extract::{FromRequestParts, State},
|
||||
http::{header::SET_COOKIE, request::Parts, HeaderMap, HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Extension, Json,
|
||||
extract::State,
|
||||
http::{HeaderMap, HeaderValue, StatusCode, header::SET_COOKIE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
admin::{
|
||||
auth::service::AdminSessionError,
|
||||
response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
api::{
|
||||
AppState,
|
||||
admin::{
|
||||
response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
session::{ADMIN_SESSION_COOKIE, admin_session_cookie},
|
||||
},
|
||||
middleware::request_id::ClientIp,
|
||||
},
|
||||
http::middleware::request_id::ClientIp,
|
||||
auth::types::SessionError,
|
||||
infra::time::china_rfc3339,
|
||||
runtime::state::AppState,
|
||||
};
|
||||
|
||||
const ADMIN_SESSION_COOKIE: &str = "cpr_admin_session";
|
||||
const ADMIN_SESSION_COOKIE_ATTRS: &str = "Path=/; Secure; HttpOnly; SameSite=Lax";
|
||||
|
||||
/// 已通过管理员会话或管理员 API Key 鉴权的请求。
|
||||
pub(crate) struct AdminAuth;
|
||||
|
||||
impl FromRequestParts<AppState> for AdminAuth {
|
||||
type Rejection = AdminError;
|
||||
|
||||
async fn from_request_parts(
|
||||
parts: &mut Parts,
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
require_admin_auth(state, &parts.headers).await?;
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct AdminLoginRequest {
|
||||
@@ -74,7 +61,7 @@ pub(crate) async fn login(
|
||||
let session = match login_result {
|
||||
Ok(Some(session)) => session,
|
||||
Ok(None) => return Err(AdminError::invalid_admin_credentials()),
|
||||
Err(AdminSessionError::LoginThrottled) => return Err(AdminError::too_many_login_attempts()),
|
||||
Err(SessionError::LoginThrottled) => return Err(AdminError::too_many_login_attempts()),
|
||||
Err(_) => return Err(AdminError::internal("Failed to create admin session")),
|
||||
};
|
||||
|
||||
@@ -120,12 +107,14 @@ pub(crate) async fn logout(
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response, AdminError> {
|
||||
if let Some(session_id) = admin_session_cookie(&headers) {
|
||||
let _ = state
|
||||
if let Some(session_id) = admin_session_cookie(&headers)
|
||||
&& let Err(error) = state
|
||||
.services
|
||||
.admin_sessions
|
||||
.delete_session(&session_id)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(error = %error, "failed to revoke admin session during logout");
|
||||
}
|
||||
|
||||
let mut response = AdminResponse::new(
|
||||
@@ -146,52 +135,6 @@ pub(crate) async fn logout(
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// 要求请求携带有效管理员会话。
|
||||
pub(crate) async fn require_admin_session(
|
||||
state: &AppState,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<(), AdminError> {
|
||||
match state
|
||||
.services
|
||||
.admin_sessions
|
||||
.validate(admin_session_cookie(headers).as_deref())
|
||||
.await
|
||||
{
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(AdminError::admin_session_required()),
|
||||
Err(_) => Err(AdminError::internal("Failed to validate admin session")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 要求请求携带有效管理员凭证:会话 Cookie 或管理员 API Key。
|
||||
pub(crate) async fn require_admin_auth(
|
||||
state: &AppState,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<(), AdminError> {
|
||||
if let Some(api_key) = admin_api_key_header(headers) {
|
||||
return match state.services.settings.verify_admin_api_key(&api_key).await {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(AdminError::invalid_admin_api_key()),
|
||||
Err(_) => Err(AdminError::internal("Failed to validate admin API key")),
|
||||
};
|
||||
}
|
||||
|
||||
require_admin_session(state, headers).await
|
||||
}
|
||||
|
||||
fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
|
||||
let cookie = headers.get("cookie")?.to_str().ok()?;
|
||||
cookie.split(';').find_map(|part| {
|
||||
let (name, value) = part.trim().split_once('=')?;
|
||||
(name == ADMIN_SESSION_COOKIE).then(|| value.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
fn admin_api_key_header(headers: &HeaderMap) -> Option<String> {
|
||||
let value = headers.get("x-api-key")?.to_str().ok()?.trim();
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
}
|
||||
|
||||
fn optional_client_ip(client_ip: Option<Extension<ClientIp>>) -> Option<String> {
|
||||
let Extension(client_ip) = client_ip?;
|
||||
Some(client_ip.as_str().to_string())
|
||||
@@ -0,0 +1,616 @@
|
||||
//! 管理端 Dashboard 聚合视图。
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use axum::{
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
session::AdminAuth,
|
||||
usage_routes::{
|
||||
UsageRecordBillingData, UsageRecordTokenDetailsData, usage_billing,
|
||||
usage_record_models, usage_token_details,
|
||||
},
|
||||
},
|
||||
fleet::{
|
||||
account::{Account, AccountStatus},
|
||||
pool::AccountCapacitySummary,
|
||||
refresh::token_refresh_status_eligible,
|
||||
},
|
||||
infra::{
|
||||
format::{format_duration_ms, format_tokens},
|
||||
time::{
|
||||
china_datetime, china_datetime_rfc3339_str, china_day_start, china_quarter_hour_start,
|
||||
china_relative_time,
|
||||
},
|
||||
},
|
||||
telemetry::{
|
||||
account_usage::query::AccountUsageTimeBucket,
|
||||
dashboard::{
|
||||
DashboardAccountCounts, DashboardCardsData, DashboardHealthTimelineData,
|
||||
DashboardTrendData, DashboardTrendKind, dashboard_cards,
|
||||
dashboard_health_timeline_data, dashboard_trend_data,
|
||||
},
|
||||
usage::{
|
||||
query::{UsageQueryFilter, UsageRecordAccountUsage},
|
||||
types::{UsageRecord, metadata_string},
|
||||
},
|
||||
},
|
||||
upstream::openai::fingerprint::Fingerprint,
|
||||
};
|
||||
|
||||
const DASHBOARD_ACCOUNT_USAGE_LIMIT: u32 = 4;
|
||||
const DASHBOARD_USAGE_RECORD_LIMIT: u32 = 10;
|
||||
const DASHBOARD_TIME_BUCKET_MINUTES: i64 = 15;
|
||||
const DASHBOARD_TIME_BUCKET_SLOTS: i64 = 7 * 24 * 4;
|
||||
const FIVE_HOUR_WINDOW_SECONDS: u64 = 18_000;
|
||||
const WEEK_WINDOW_SECONDS: u64 = 604_800;
|
||||
const MONTH_WINDOW_SECONDS: u64 = 2_592_000;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct DashboardTrendQuery {
|
||||
kind: Option<DashboardTrendKind>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DashboardSummaryData {
|
||||
cards: DashboardCardsData,
|
||||
trend: DashboardTrendData,
|
||||
health_timeline: DashboardHealthTimelineData,
|
||||
account_usage: Vec<DashboardAccountUsageData>,
|
||||
service_statuses: Vec<DashboardServiceStatusData>,
|
||||
usage_records: Vec<DashboardUsageRecordData>,
|
||||
pool_summary: AccountPoolDiagnostics,
|
||||
capacity_info: AccountCapacityDiagnostics,
|
||||
rotation_strategy: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DashboardAccountUsageData {
|
||||
id: String,
|
||||
email: String,
|
||||
plan_type: Option<String>,
|
||||
tokens: String,
|
||||
quota_used_percent: Option<f64>,
|
||||
last_used: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DashboardUsageRecordData {
|
||||
id: String,
|
||||
route: Option<String>,
|
||||
model: String,
|
||||
status_code: i64,
|
||||
transport: Option<String>,
|
||||
created_at_display: String,
|
||||
account_email: Option<String>,
|
||||
requested_model: Option<String>,
|
||||
upstream_model: Option<String>,
|
||||
client_ip: Option<String>,
|
||||
user_agent: Option<String>,
|
||||
reasoning_effort: Option<String>,
|
||||
compact: bool,
|
||||
request_kind: Option<String>,
|
||||
subagent_kind: Option<String>,
|
||||
token_details: UsageRecordTokenDetailsData,
|
||||
billing: Option<UsageRecordBillingData>,
|
||||
first_token_latency_ms_display: String,
|
||||
latency_ms_display: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DashboardServiceStatusData {
|
||||
label: String,
|
||||
value: String,
|
||||
detail: String,
|
||||
tone: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountPoolDiagnostics {
|
||||
total: usize,
|
||||
active: usize,
|
||||
expired: usize,
|
||||
quota_exhausted: usize,
|
||||
refreshing: usize,
|
||||
disabled: usize,
|
||||
banned: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountCapacityDiagnostics {
|
||||
max_concurrent_per_account: usize,
|
||||
total_slots: usize,
|
||||
used_slots: usize,
|
||||
available_slots: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct FingerprintDiagnostics {
|
||||
source: &'static str,
|
||||
originator: String,
|
||||
app_version: String,
|
||||
build_number: String,
|
||||
platform: String,
|
||||
arch: String,
|
||||
chromium_version: String,
|
||||
user_agent: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
updated_at: Option<String>,
|
||||
}
|
||||
|
||||
fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics {
|
||||
FingerprintDiagnostics {
|
||||
source: "runtime",
|
||||
originator: fingerprint.originator.clone(),
|
||||
app_version: fingerprint.app_version.clone(),
|
||||
build_number: fingerprint.build_number.clone(),
|
||||
platform: fingerprint.platform.clone(),
|
||||
arch: fingerprint.arch.clone(),
|
||||
chromium_version: fingerprint.chromium_version.clone(),
|
||||
user_agent: fingerprint.user_agent(),
|
||||
updated_at: fingerprint.updated_at.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AccountCapacitySummary> for AccountCapacityDiagnostics {
|
||||
fn from(summary: AccountCapacitySummary) -> Self {
|
||||
Self {
|
||||
max_concurrent_per_account: summary.max_concurrent_per_account,
|
||||
total_slots: summary.total_slots,
|
||||
used_slots: summary.used_slots,
|
||||
available_slots: summary.available_slots,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum QuotaWindowPriority {
|
||||
FiveHour,
|
||||
Weekly,
|
||||
Monthly,
|
||||
Other,
|
||||
}
|
||||
|
||||
/// `GET /api/admin/dashboard/summary`
|
||||
pub(crate) async fn dashboard_summary(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<DashboardTrendQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let accounts = state
|
||||
.services
|
||||
.accounts
|
||||
.list_pool_accounts()
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("accounts", &error))?;
|
||||
let capacity = state.services.account_pool.capacity_summary_now().await;
|
||||
let now = Utc::now();
|
||||
let today_filter = today_usage_record_filter(now);
|
||||
let lifetime_usage = state
|
||||
.services
|
||||
.usage
|
||||
.summary()
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("account usage summary", &error))?;
|
||||
let account_usage_records = state
|
||||
.services
|
||||
.usage_records
|
||||
.account_usage(today_filter.clone(), DASHBOARD_ACCOUNT_USAGE_LIMIT)
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("account usage ranking", &error))?;
|
||||
let recent_events = state
|
||||
.services
|
||||
.usage_records
|
||||
.list_recent(DASHBOARD_USAGE_RECORD_LIMIT, today_filter)
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("recent usage records", &error))?;
|
||||
|
||||
let time_buckets = dashboard_time_buckets(&state, now).await?;
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let refreshing_account_ids = state
|
||||
.services
|
||||
.token_refresh
|
||||
.refreshing_account_ids(&account_ids, now)
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("refreshing accounts", &error))?;
|
||||
let pool_summary = account_pool_summary(&accounts, &refreshing_account_ids);
|
||||
let trend = dashboard_trend_data(&time_buckets, query.kind.unwrap_or_default());
|
||||
let quota_used_by_account =
|
||||
account_quota_used_percent_by_id(&state, &account_usage_records).await?;
|
||||
let settings = state.services.settings.current();
|
||||
let recent_usage_records = recent_events;
|
||||
let account_emails = state
|
||||
.services
|
||||
.usage_records
|
||||
.account_email_map(&recent_usage_records)
|
||||
.await
|
||||
.map_err(|error| AdminError::usage_record_accounts_failed(error.to_string()))?;
|
||||
let dashboard_usage_records =
|
||||
dashboard_usage_record_items(recent_usage_records, &account_emails);
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DashboardSummaryData {
|
||||
cards: dashboard_cards(
|
||||
dashboard_account_counts(&accounts),
|
||||
&time_buckets,
|
||||
&lifetime_usage,
|
||||
),
|
||||
trend,
|
||||
health_timeline: dashboard_health_timeline_data(&time_buckets),
|
||||
account_usage: account_usage_data(
|
||||
&accounts,
|
||||
&account_usage_records,
|
||||
"a_used_by_account,
|
||||
now,
|
||||
),
|
||||
service_statuses: service_status_data(&state),
|
||||
usage_records: dashboard_usage_records,
|
||||
pool_summary,
|
||||
capacity_info: AccountCapacityDiagnostics::from(capacity),
|
||||
rotation_strategy: Some(settings.rotation_strategy.clone()),
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
fn today_usage_record_filter(now: DateTime<Utc>) -> UsageQueryFilter {
|
||||
UsageQueryFilter {
|
||||
start_time: Some(china_day_start(now)),
|
||||
end_time: Some(now),
|
||||
..UsageQueryFilter::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/admin/dashboard/trend?kind=usage|latency|errors`
|
||||
pub(crate) async fn dashboard_trend(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<DashboardTrendQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let time_buckets = dashboard_time_buckets(&state, Utc::now()).await?;
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(dashboard_trend_data(
|
||||
&time_buckets,
|
||||
query.kind.unwrap_or_default(),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn dashboard_time_buckets(
|
||||
state: &AppState,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Vec<AccountUsageTimeBucket>, AdminError> {
|
||||
let current_slot = china_quarter_hour_start(now);
|
||||
let start = current_slot
|
||||
- Duration::minutes(DASHBOARD_TIME_BUCKET_MINUTES * (DASHBOARD_TIME_BUCKET_SLOTS - 1));
|
||||
state
|
||||
.services
|
||||
.usage
|
||||
.time_buckets(start, now)
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("time buckets", &error))
|
||||
}
|
||||
|
||||
fn dashboard_account_counts(accounts: &[Account]) -> DashboardAccountCounts {
|
||||
DashboardAccountCounts {
|
||||
total: accounts.len() as u64,
|
||||
enabled: accounts
|
||||
.iter()
|
||||
.filter(|account| account.status == AccountStatus::Active)
|
||||
.count() as u64,
|
||||
abnormal: accounts
|
||||
.iter()
|
||||
.filter(|account| account.status != AccountStatus::Active)
|
||||
.count() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
fn account_pool_summary(
|
||||
accounts: &[Account],
|
||||
refreshing_account_ids: &HashSet<String>,
|
||||
) -> AccountPoolDiagnostics {
|
||||
let mut summary = AccountPoolDiagnostics {
|
||||
total: accounts.len(),
|
||||
..AccountPoolDiagnostics::default()
|
||||
};
|
||||
for account in accounts {
|
||||
match account.status {
|
||||
status
|
||||
if token_refresh_status_eligible(status)
|
||||
&& refreshing_account_ids.contains(&account.id) =>
|
||||
{
|
||||
summary.refreshing += 1;
|
||||
}
|
||||
AccountStatus::Active => summary.active += 1,
|
||||
AccountStatus::Expired => summary.expired += 1,
|
||||
AccountStatus::QuotaExhausted => summary.quota_exhausted += 1,
|
||||
AccountStatus::Disabled => summary.disabled += 1,
|
||||
AccountStatus::Banned => summary.banned += 1,
|
||||
}
|
||||
}
|
||||
summary
|
||||
}
|
||||
|
||||
fn account_usage_data(
|
||||
accounts: &[Account],
|
||||
usage_records: &[UsageRecordAccountUsage],
|
||||
quota_used_by_account: &HashMap<String, f64>,
|
||||
now: DateTime<Utc>,
|
||||
) -> Vec<DashboardAccountUsageData> {
|
||||
let account_by_id = accounts
|
||||
.iter()
|
||||
.map(|account| (account.id.as_str(), account))
|
||||
.collect::<HashMap<_, _>>();
|
||||
usage_records
|
||||
.iter()
|
||||
.map(|usage| {
|
||||
let account = account_by_id.get(usage.account_id.as_str()).copied();
|
||||
let quota_used_percent = quota_used_by_account
|
||||
.get(&usage.account_id)
|
||||
.copied()
|
||||
.or_else(|| {
|
||||
matches!(
|
||||
account.map(|account| account.status),
|
||||
Some(AccountStatus::QuotaExhausted)
|
||||
)
|
||||
.then_some(100.0)
|
||||
});
|
||||
DashboardAccountUsageData {
|
||||
id: usage.account_id.clone(),
|
||||
email: account
|
||||
.and_then(|account| account.email.as_ref())
|
||||
.cloned()
|
||||
.unwrap_or_else(|| usage.account_id.clone()),
|
||||
plan_type: account
|
||||
.and_then(|account| account.plan_type.as_ref())
|
||||
.cloned(),
|
||||
tokens: format_tokens(usage.total_tokens),
|
||||
quota_used_percent,
|
||||
last_used: china_relative_time(Some(usage.last_used_at), now),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn account_quota_used_percent_by_id(
|
||||
state: &AppState,
|
||||
usage_records: &[UsageRecordAccountUsage],
|
||||
) -> Result<HashMap<String, f64>, AdminError> {
|
||||
let mut quota_used_by_account = HashMap::with_capacity(usage_records.len());
|
||||
for usage in usage_records {
|
||||
let quota_json = state
|
||||
.services
|
||||
.accounts
|
||||
.get_quota_json(&usage.account_id)
|
||||
.await
|
||||
.map_err(|error| dashboard_data_error("account quota", &error))?;
|
||||
let Some(quota_json) = quota_json else {
|
||||
continue;
|
||||
};
|
||||
if let Some(used_percent) = quota_used_percent("a_json) {
|
||||
quota_used_by_account.insert(usage.account_id.clone(), used_percent);
|
||||
}
|
||||
}
|
||||
Ok(quota_used_by_account)
|
||||
}
|
||||
|
||||
fn dashboard_data_error(source: &'static str, error: &impl std::fmt::Display) -> AdminError {
|
||||
tracing::error!(source, error = %error, "failed to load dashboard data");
|
||||
AdminError::internal("Failed to load dashboard data")
|
||||
}
|
||||
|
||||
fn quota_used_percent(quota_json: &str) -> Option<f64> {
|
||||
let quota = serde_json::from_str::<Value>(quota_json).ok()?;
|
||||
let mut selected = None;
|
||||
if let Some(monthly_limit) = quota.get("monthly_limit") {
|
||||
select_quota_used_percent(
|
||||
&mut selected,
|
||||
quota_window_priority(monthly_limit, QuotaWindowPriority::Monthly),
|
||||
monthly_limit.get("used_percent").and_then(percent_value),
|
||||
);
|
||||
}
|
||||
if let Some(snapshots) = quota.get("snapshots").and_then(Value::as_array) {
|
||||
for snapshot in snapshots {
|
||||
for role in ["primary", "secondary"] {
|
||||
if let Some(window) = snapshot.get(role) {
|
||||
select_quota_used_percent(
|
||||
&mut selected,
|
||||
quota_window_priority(window, QuotaWindowPriority::Other),
|
||||
window.get("used_percent").and_then(percent_value),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.map(|(_, used_percent)| used_percent)
|
||||
}
|
||||
|
||||
fn select_quota_used_percent(
|
||||
selected: &mut Option<(QuotaWindowPriority, f64)>,
|
||||
priority: QuotaWindowPriority,
|
||||
used_percent: Option<f64>,
|
||||
) {
|
||||
let Some(used_percent) = used_percent else {
|
||||
return;
|
||||
};
|
||||
match selected {
|
||||
Some((selected_priority, selected_percent))
|
||||
if priority > *selected_priority
|
||||
|| (priority == *selected_priority && used_percent <= *selected_percent) => {}
|
||||
_ => *selected = Some((priority, used_percent)),
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_window_priority(window: &Value, fallback: QuotaWindowPriority) -> QuotaWindowPriority {
|
||||
let window_seconds = window
|
||||
.get("window_minutes")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|minutes| minutes.checked_mul(60));
|
||||
match window_seconds {
|
||||
Some(seconds) if quota_window_matches(seconds, FIVE_HOUR_WINDOW_SECONDS) => {
|
||||
QuotaWindowPriority::FiveHour
|
||||
}
|
||||
Some(seconds) if quota_window_matches(seconds, WEEK_WINDOW_SECONDS) => {
|
||||
QuotaWindowPriority::Weekly
|
||||
}
|
||||
Some(seconds) if quota_window_matches(seconds, MONTH_WINDOW_SECONDS) => {
|
||||
QuotaWindowPriority::Monthly
|
||||
}
|
||||
_ => fallback,
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_window_matches(actual: u64, expected: u64) -> bool {
|
||||
actual > 0 && actual.abs_diff(expected) <= expected / 20
|
||||
}
|
||||
|
||||
fn percent_value(value: &Value) -> Option<f64> {
|
||||
let percent = value
|
||||
.as_f64()
|
||||
.or_else(|| value.as_str().and_then(|value| value.parse::<f64>().ok()))?;
|
||||
percent.is_finite().then_some(percent.clamp(0.0, 100.0))
|
||||
}
|
||||
|
||||
fn dashboard_usage_record_items(
|
||||
items: Vec<UsageRecord>,
|
||||
account_emails: &HashMap<String, String>,
|
||||
) -> Vec<DashboardUsageRecordData> {
|
||||
items
|
||||
.into_iter()
|
||||
.map(|record| dashboard_usage_record_data(record, account_emails))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn dashboard_usage_record_data(
|
||||
record: UsageRecord,
|
||||
account_emails: &HashMap<String, String>,
|
||||
) -> DashboardUsageRecordData {
|
||||
let account_email = account_emails
|
||||
.get(&record.account_id)
|
||||
.cloned()
|
||||
.or_else(|| metadata_string(&record.metadata, &["accountEmail", "account_email"]));
|
||||
let (requested_model, upstream_model) = usage_record_models(&record);
|
||||
let client_ip = metadata_string(&record.metadata, &["clientIp", "ipAddress", "ip_address"]);
|
||||
let user_agent = metadata_string(&record.metadata, &["userAgent", "user_agent"]);
|
||||
let reasoning_effort =
|
||||
metadata_string(&record.metadata, &["reasoningEffort", "reasoning_effort"]);
|
||||
let compact = record
|
||||
.metadata
|
||||
.get("compact")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let request_kind = metadata_string(&record.metadata, &["requestKind", "request_kind"]);
|
||||
let subagent_kind = metadata_string(&record.metadata, &["subagentKind", "subagent_kind"]);
|
||||
let token_details = usage_token_details(&record);
|
||||
let billing = usage_billing(&record, upstream_model.as_deref(), &token_details);
|
||||
let first_token_latency_ms = record.first_token_ms;
|
||||
|
||||
DashboardUsageRecordData {
|
||||
id: record.id,
|
||||
route: record.route,
|
||||
model: record.model,
|
||||
status_code: record.status_code,
|
||||
transport: record.transport,
|
||||
created_at_display: china_datetime(&record.created_at),
|
||||
account_email,
|
||||
requested_model,
|
||||
upstream_model,
|
||||
client_ip,
|
||||
user_agent,
|
||||
reasoning_effort,
|
||||
compact,
|
||||
request_kind,
|
||||
subagent_kind,
|
||||
token_details,
|
||||
billing,
|
||||
first_token_latency_ms_display: format_duration_ms(first_token_latency_ms),
|
||||
latency_ms_display: format_duration_ms(record.latency_ms),
|
||||
}
|
||||
}
|
||||
|
||||
fn service_status_data(state: &AppState) -> Vec<DashboardServiceStatusData> {
|
||||
let current_fingerprint = state.services.fingerprint.snapshot();
|
||||
let fingerprint = fingerprint_diagnostics(¤t_fingerprint);
|
||||
vec![
|
||||
DashboardServiceStatusData {
|
||||
label: "客户端版本".to_string(),
|
||||
value: empty_dash(fingerprint.app_version),
|
||||
detail: if fingerprint.build_number.trim().is_empty() {
|
||||
"-".to_string()
|
||||
} else {
|
||||
format!("Build {}", fingerprint.build_number)
|
||||
},
|
||||
tone: "info".to_string(),
|
||||
},
|
||||
DashboardServiceStatusData {
|
||||
label: "平台架构".to_string(),
|
||||
value: empty_dash(fingerprint.platform),
|
||||
detail: empty_dash(fingerprint.arch),
|
||||
tone: "info".to_string(),
|
||||
},
|
||||
DashboardServiceStatusData {
|
||||
label: "Chromium".to_string(),
|
||||
value: if fingerprint.chromium_version.trim().is_empty() {
|
||||
"-".to_string()
|
||||
} else {
|
||||
format!("v{}", fingerprint.chromium_version)
|
||||
},
|
||||
detail: empty_dash(fingerprint.originator),
|
||||
tone: "normal".to_string(),
|
||||
},
|
||||
DashboardServiceStatusData {
|
||||
label: "更新时间".to_string(),
|
||||
value: format_fingerprint_updated_at(fingerprint.updated_at),
|
||||
detail: String::new(),
|
||||
tone: "normal".to_string(),
|
||||
},
|
||||
DashboardServiceStatusData {
|
||||
label: "User Agent".to_string(),
|
||||
value: fingerprint.user_agent,
|
||||
detail: String::new(),
|
||||
tone: "normal".to_string(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn empty_dash(value: String) -> String {
|
||||
if value.trim().is_empty() {
|
||||
"-".to_string()
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
fn format_fingerprint_updated_at(value: Option<String>) -> String {
|
||||
let Some(value) = value else {
|
||||
return "-".to_string();
|
||||
};
|
||||
let value = value.trim();
|
||||
if value.is_empty() {
|
||||
return "-".to_string();
|
||||
}
|
||||
china_datetime_rfc3339_str(value)
|
||||
}
|
||||
@@ -1,33 +1,38 @@
|
||||
//! 管理端 v1 接口访问 Key 处理器(列表、创建、生命周期)。
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
Json,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
admin::auth::session::AdminAuth,
|
||||
admin::keys::service::{AdminClientApiKey, AdminClientKeyError},
|
||||
admin::response::{
|
||||
api::AppState,
|
||||
api::admin::response::{
|
||||
AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse, BatchDeleteData,
|
||||
EditableUpdateMessages, parse_editable_update,
|
||||
},
|
||||
admin::update_payload::{parse_editable_update, EditableUpdateMessages},
|
||||
api::admin::session::AdminAuth,
|
||||
infra::{
|
||||
json::{clamp_limit, Page},
|
||||
json::{NumberedPage, SortDirection, clamp_limit, clamp_page},
|
||||
time::{china_relative_time_str, china_rfc3339_str},
|
||||
},
|
||||
runtime::state::AppState,
|
||||
keys::types::{
|
||||
ClientApiKeyListSort, ClientApiKeySortField, KeyManageError, ManagedClientApiKey,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct ApiKeysQuery {
|
||||
cursor: Option<String>,
|
||||
limit: Option<u32>,
|
||||
page: Option<u32>,
|
||||
page_size: Option<u32>,
|
||||
search: Option<String>,
|
||||
sort_by: Option<String>,
|
||||
sort_direction: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -57,8 +62,8 @@ struct ClientApiKeyData {
|
||||
last_used_at_display: String,
|
||||
}
|
||||
|
||||
impl From<AdminClientApiKey> for ClientApiKeyData {
|
||||
fn from(k: AdminClientApiKey) -> Self {
|
||||
impl From<ManagedClientApiKey> for ClientApiKeyData {
|
||||
fn from(k: ManagedClientApiKey) -> Self {
|
||||
Self {
|
||||
id: k.id,
|
||||
name: k.name,
|
||||
@@ -74,12 +79,12 @@ impl From<AdminClientApiKey> for ClientApiKeyData {
|
||||
}
|
||||
}
|
||||
|
||||
fn client_key_error(error: &AdminClientKeyError) -> AdminError {
|
||||
fn client_key_error(error: &KeyManageError) -> AdminError {
|
||||
match error {
|
||||
AdminClientKeyError::InvalidStatus(_)
|
||||
| AdminClientKeyError::EmptyName
|
||||
| AdminClientKeyError::EmptyIds
|
||||
| AdminClientKeyError::LabelTooLong => AdminError::bad_request(error.to_string()),
|
||||
KeyManageError::InvalidStatus(_)
|
||||
| KeyManageError::EmptyName
|
||||
| KeyManageError::EmptyIds
|
||||
| KeyManageError::LabelTooLong => AdminError::bad_request(error.to_string()),
|
||||
_ => AdminError::internal(error.to_string()),
|
||||
}
|
||||
}
|
||||
@@ -117,27 +122,56 @@ pub(crate) async fn api_keys(
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<ApiKeysQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let limit = clamp_limit(query.limit.unwrap_or(50));
|
||||
let page = clamp_page(query.page.unwrap_or(1));
|
||||
let page_size = clamp_limit(query.page_size.unwrap_or(50));
|
||||
let sort = client_api_key_list_sort(query.sort_by, query.sort_direction)?;
|
||||
match state
|
||||
.services
|
||||
.admin_client_keys
|
||||
.list(query.cursor, limit)
|
||||
.list_page(page, page_size, query.search, sort)
|
||||
.await
|
||||
{
|
||||
Ok(page) => {
|
||||
let page = Page {
|
||||
let page = NumberedPage {
|
||||
items: page.items.into_iter().map(ClientApiKeyData::from).collect(),
|
||||
next_cursor: page.next_cursor,
|
||||
total: page.total,
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
};
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminPageEnvelope::ok(page, limit),
|
||||
AdminPageEnvelope::ok(page),
|
||||
))
|
||||
}
|
||||
Err(error) => Err(client_key_error(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn client_api_key_list_sort(
|
||||
sort_by: Option<String>,
|
||||
sort_direction: Option<String>,
|
||||
) -> Result<Option<ClientApiKeyListSort>, AdminError> {
|
||||
let (sort_by, sort_direction) = match (sort_by, sort_direction) {
|
||||
(None, None) => return Ok(None),
|
||||
(Some(sort_by), Some(sort_direction)) => (sort_by, sort_direction),
|
||||
_ => {
|
||||
return Err(AdminError::bad_request(
|
||||
"Client API key sort field and direction must be provided together",
|
||||
));
|
||||
}
|
||||
};
|
||||
let field = match sort_by.trim() {
|
||||
"name" => ClientApiKeySortField::Name,
|
||||
"enabled" => ClientApiKeySortField::Enabled,
|
||||
"createdAt" => ClientApiKeySortField::CreatedAt,
|
||||
"lastUsedAt" => ClientApiKeySortField::LastUsedAt,
|
||||
_ => return Err(AdminError::bad_request("Invalid client API key sort field")),
|
||||
};
|
||||
let direction = SortDirection::parse(&sort_direction)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid client API key sort direction"))?;
|
||||
Ok(Some(ClientApiKeyListSort { field, direction }))
|
||||
}
|
||||
|
||||
/// `POST /api/admin/keys`
|
||||
pub(crate) async fn create_api_key(
|
||||
State(state): State<AppState>,
|
||||
@@ -0,0 +1,13 @@
|
||||
//! 管理端 HTTP API。
|
||||
|
||||
pub mod accounts_routes;
|
||||
pub mod auth_routes;
|
||||
pub mod dashboard_routes;
|
||||
pub mod keys_routes;
|
||||
pub mod ops_routes;
|
||||
pub mod response;
|
||||
pub mod router;
|
||||
pub mod session;
|
||||
pub mod settings_routes;
|
||||
pub mod system_routes;
|
||||
pub mod usage_routes;
|
||||
@@ -0,0 +1,135 @@
|
||||
//! 运维错误明细 HTTP 处理器。
|
||||
|
||||
use axum::{
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{AdminError, AdminPageEnvelope, AdminResponse},
|
||||
session::AdminAuth,
|
||||
},
|
||||
infra::{
|
||||
json::{NumberedPage, clamp_limit, clamp_page},
|
||||
time::china_datetime,
|
||||
},
|
||||
telemetry::{ops::query::OpsQueryError, ops::store::OpsErrorFilter, ops::types::OpsErrorLog},
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct OpsErrorsQuery {
|
||||
page: Option<u32>,
|
||||
page_size: Option<u32>,
|
||||
kind: Option<String>,
|
||||
client_api_key_id: Option<String>,
|
||||
provider: Option<String>,
|
||||
request_id: Option<String>,
|
||||
account_id: Option<String>,
|
||||
route: Option<String>,
|
||||
model: Option<String>,
|
||||
status_code: Option<i64>,
|
||||
client_status_code: Option<i64>,
|
||||
upstream_status_code: Option<i64>,
|
||||
transport: Option<String>,
|
||||
attempt_index: Option<i64>,
|
||||
failure_class: Option<String>,
|
||||
response_id: Option<String>,
|
||||
upstream_request_id: Option<String>,
|
||||
search: Option<String>,
|
||||
start_time: Option<String>,
|
||||
end_time: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct OpsErrorData {
|
||||
#[serde(flatten)]
|
||||
error: OpsErrorLog,
|
||||
created_at_display: String,
|
||||
}
|
||||
|
||||
/// `GET /api/admin/ops/errors`
|
||||
pub(crate) async fn ops_errors(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<OpsErrorsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let page = clamp_page(query.page.unwrap_or(1));
|
||||
let page_size = clamp_limit(query.page_size.unwrap_or(50));
|
||||
let filter = filter_from_query(query)?;
|
||||
|
||||
state
|
||||
.services
|
||||
.ops_errors
|
||||
.list_page(page, page_size, filter)
|
||||
.await
|
||||
.map(|page| {
|
||||
let page = NumberedPage {
|
||||
items: ops_error_items(page.items),
|
||||
total: page.total,
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
};
|
||||
AdminResponse::new(StatusCode::OK, AdminPageEnvelope::ok(page))
|
||||
})
|
||||
.map_err(log_error)
|
||||
}
|
||||
|
||||
fn filter_from_query(query: OpsErrorsQuery) -> Result<OpsErrorFilter, AdminError> {
|
||||
Ok(OpsErrorFilter {
|
||||
kind: non_empty(query.kind),
|
||||
client_api_key_id: non_empty(query.client_api_key_id),
|
||||
provider: non_empty(query.provider),
|
||||
request_id: non_empty(query.request_id),
|
||||
account_id: non_empty(query.account_id),
|
||||
route: non_empty(query.route),
|
||||
model: non_empty(query.model),
|
||||
status_code: query.status_code,
|
||||
client_status_code: query.client_status_code,
|
||||
upstream_status_code: query.upstream_status_code,
|
||||
transport: non_empty(query.transport),
|
||||
attempt_index: query.attempt_index,
|
||||
failure_class: non_empty(query.failure_class),
|
||||
response_id: non_empty(query.response_id),
|
||||
upstream_request_id: non_empty(query.upstream_request_id),
|
||||
search: non_empty(query.search),
|
||||
start_time: optional_datetime(query.start_time)?,
|
||||
end_time: optional_datetime(query.end_time)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn ops_error_items(items: Vec<OpsErrorLog>) -> Vec<OpsErrorData> {
|
||||
items
|
||||
.into_iter()
|
||||
.map(|error| OpsErrorData {
|
||||
created_at_display: china_datetime(&error.created_at),
|
||||
error,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn optional_datetime(
|
||||
value: Option<String>,
|
||||
) -> Result<Option<chrono::DateTime<chrono::Utc>>, AdminError> {
|
||||
let Some(value) = non_empty(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
chrono::DateTime::parse_from_rfc3339(&value)
|
||||
.map(|value| Some(value.with_timezone(&chrono::Utc)))
|
||||
.map_err(|_| AdminError::invalid_time_range("Invalid time range"))
|
||||
}
|
||||
|
||||
fn non_empty(value: Option<String>) -> Option<String> {
|
||||
value
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn log_error(error: OpsQueryError) -> AdminError {
|
||||
AdminError::internal(error.to_string())
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
//! 管理端响应封装与 From 转换。
|
||||
|
||||
use crate::infra::json::{total_pages, NumberedPage, Page};
|
||||
use crate::infra::json::{NumberedPage, total_pages};
|
||||
use axum::{
|
||||
Json,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::fmt;
|
||||
|
||||
pub(crate) const ADMIN_OK_CODE: u32 = 200;
|
||||
@@ -15,7 +16,6 @@ const ADMIN_MALFORMED_JSON_CODE: u32 = 40000;
|
||||
const ADMIN_BAD_REQUEST_CODE: u32 = 40001;
|
||||
const ADMIN_INVALID_TIME_RANGE_CODE: u32 = 40002;
|
||||
const ADMIN_INVALID_MODEL_SOURCE_CODE: u32 = 40003;
|
||||
const ADMIN_INVALID_ENDPOINT_SOURCE_CODE: u32 = 40004;
|
||||
const ADMIN_SESSION_REQUIRED_CODE: u32 = 40101;
|
||||
const ADMIN_INVALID_CREDENTIALS_CODE: u32 = 40102;
|
||||
const ADMIN_INVALID_API_KEY_CODE: u32 = 40103;
|
||||
@@ -67,14 +67,6 @@ impl AdminError {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn invalid_endpoint_source(message: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
StatusCode::BAD_REQUEST,
|
||||
ADMIN_INVALID_ENDPOINT_SOURCE_CODE,
|
||||
message,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn admin_session_required() -> Self {
|
||||
Self::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
@@ -186,33 +178,16 @@ impl<T> AdminEnvelope<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// 分页元数据。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct CursorPageMeta {
|
||||
pub(crate) limit: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
/// 页码分页元数据。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct NumberedPageMeta {
|
||||
pub(crate) struct PageMeta {
|
||||
pub(crate) page: u32,
|
||||
pub(crate) page_size: u32,
|
||||
pub(crate) total: u64,
|
||||
pub(crate) total_pages: u32,
|
||||
}
|
||||
|
||||
/// 分页元数据。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum PageMeta {
|
||||
Cursor(CursorPageMeta),
|
||||
Numbered(NumberedPageMeta),
|
||||
}
|
||||
|
||||
/// 分页响应信封。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -239,19 +214,7 @@ pub(crate) struct BatchDeleteData {
|
||||
}
|
||||
|
||||
impl<T> AdminPageEnvelope<T> {
|
||||
pub fn ok(page: Page<T>, limit: u32) -> Self {
|
||||
let Page { items, next_cursor } = page;
|
||||
Self {
|
||||
code: ADMIN_OK_CODE,
|
||||
message: ADMIN_OK_MESSAGE.into(),
|
||||
data: PageData {
|
||||
items,
|
||||
page: PageMeta::Cursor(CursorPageMeta { limit, next_cursor }),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn numbered(page: NumberedPage<T>) -> Self {
|
||||
pub fn ok(page: NumberedPage<T>) -> Self {
|
||||
let NumberedPage {
|
||||
items,
|
||||
total,
|
||||
@@ -263,12 +226,12 @@ impl<T> AdminPageEnvelope<T> {
|
||||
message: ADMIN_OK_MESSAGE.into(),
|
||||
data: PageData {
|
||||
items,
|
||||
page: PageMeta::Numbered(NumberedPageMeta {
|
||||
page: PageMeta {
|
||||
page,
|
||||
page_size,
|
||||
total,
|
||||
total_pages: total_pages(total, page_size),
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -291,3 +254,59 @@ impl<T: Serialize> IntoResponse for AdminResponse<T> {
|
||||
(self.status, Json(self.body)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct EditableUpdateRequest {
|
||||
id: String,
|
||||
label: Option<Option<String>>,
|
||||
status: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) struct ParsedEditableUpdate {
|
||||
pub id: String,
|
||||
pub label: Option<Option<String>>,
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) struct EditableUpdateMessages<'a> {
|
||||
pub object_required: &'a str,
|
||||
pub invalid: &'a str,
|
||||
pub empty_update: &'a str,
|
||||
pub unknown_field_editable: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn parse_editable_update(
|
||||
payload: &Value,
|
||||
messages: EditableUpdateMessages<'_>,
|
||||
) -> Result<ParsedEditableUpdate, AdminError> {
|
||||
let object = payload
|
||||
.as_object()
|
||||
.ok_or_else(|| AdminError::bad_request(messages.object_required))?;
|
||||
if messages.unknown_field_editable {
|
||||
for field in object.keys() {
|
||||
if !matches!(field.as_str(), "id" | "label" | "status") {
|
||||
return Err(AdminError::bad_request(format!("{field} is not editable")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let update = serde_json::from_value::<EditableUpdateRequest>(payload.clone())
|
||||
.map_err(|_| AdminError::bad_request(messages.invalid))?;
|
||||
if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) {
|
||||
return Err(AdminError::bad_request(messages.invalid));
|
||||
}
|
||||
if update.label.is_none() && update.status.is_none() {
|
||||
return Err(AdminError::bad_request(messages.empty_update));
|
||||
}
|
||||
|
||||
Ok(ParsedEditableUpdate {
|
||||
id: update.id,
|
||||
label: update.label,
|
||||
status: update.status,
|
||||
})
|
||||
}
|
||||
|
||||
fn is_blank(value: &str) -> bool {
|
||||
value.trim().is_empty()
|
||||
}
|
||||
@@ -1,36 +1,38 @@
|
||||
//! 管理端路由。
|
||||
|
||||
use axum::{
|
||||
routing::{get, post},
|
||||
Router,
|
||||
http::{HeaderValue, header},
|
||||
middleware,
|
||||
response::Response,
|
||||
routing::{get, post},
|
||||
};
|
||||
|
||||
use crate::runtime::state::AppState;
|
||||
use crate::api::AppState;
|
||||
|
||||
use super::{
|
||||
accounts::routes::{
|
||||
accounts_routes::{
|
||||
account_models, account_quota, accounts, batch_delete_accounts, create_account,
|
||||
export_accounts, health_check_accounts, import_accounts, oauth_authorize_account,
|
||||
oauth_exchange_account, refresh_account, test_account_connection, update_account,
|
||||
},
|
||||
auth::session::{login, logout, session_status},
|
||||
keys::routes::{api_keys, batch_delete_api_keys, create_api_key, update_api_key},
|
||||
monitoring::{
|
||||
dashboard::{dashboard_summary, dashboard_trend},
|
||||
usage_record_routes::{
|
||||
clear_usage_records, usage_record_detail, usage_records,
|
||||
usage_records_endpoint_distribution, usage_records_latency_trend,
|
||||
usage_records_model_distribution, usage_records_summary, usage_records_token_trend,
|
||||
},
|
||||
},
|
||||
settings::routes::{
|
||||
auth_routes::{login, logout, session_status},
|
||||
dashboard_routes::{dashboard_summary, dashboard_trend},
|
||||
keys_routes::{api_keys, batch_delete_api_keys, create_api_key, update_api_key},
|
||||
ops_routes::ops_errors,
|
||||
settings_routes::{
|
||||
admin_api_key_status, delete_admin_api_key, regenerate_admin_api_key, settings,
|
||||
update_settings,
|
||||
},
|
||||
system::routes::{
|
||||
system_routes::{
|
||||
perform_update, restart, rollback, update_detail, update_event_stream, update_status,
|
||||
version,
|
||||
},
|
||||
usage_routes::{
|
||||
clear_usage_records, usage_record_detail, usage_records,
|
||||
usage_records_endpoint_distribution, usage_records_latency_trend,
|
||||
usage_records_model_distribution, usage_records_summary, usage_records_token_trend,
|
||||
},
|
||||
};
|
||||
|
||||
/// 构造管理端路由。
|
||||
@@ -101,7 +103,16 @@ pub fn router() -> Router<AppState> {
|
||||
)
|
||||
.route("/api/admin/usage/records/delete", post(clear_usage_records))
|
||||
.route("/api/admin/usage/records/detail", get(usage_record_detail))
|
||||
.route("/api/admin/ops/errors", get(ops_errors))
|
||||
.route("/api/admin/keys", get(api_keys).post(create_api_key))
|
||||
.route("/api/admin/keys/delete", post(batch_delete_api_keys))
|
||||
.route("/api/admin/keys/update", post(update_api_key))
|
||||
.layer(middleware::map_response(no_store))
|
||||
}
|
||||
|
||||
async fn no_store(mut response: Response) -> Response {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
response
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//! 管理端认证提取器与会话鉴权。
|
||||
|
||||
use axum::{
|
||||
extract::FromRequestParts,
|
||||
http::{HeaderMap, request::Parts},
|
||||
};
|
||||
|
||||
use crate::api::{AppState, admin::response::AdminError};
|
||||
|
||||
pub(super) const ADMIN_SESSION_COOKIE: &str = "cpr_admin_session";
|
||||
|
||||
/// 已通过管理员会话或管理员 API Key 鉴权的请求。
|
||||
pub(crate) struct AdminAuth;
|
||||
|
||||
impl FromRequestParts<AppState> for AdminAuth {
|
||||
type Rejection = AdminError;
|
||||
|
||||
async fn from_request_parts(
|
||||
parts: &mut Parts,
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
require_admin_auth(state, &parts.headers).await?;
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
|
||||
/// 要求请求携带有效管理员会话。
|
||||
pub(crate) async fn require_admin_session(
|
||||
state: &AppState,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<(), AdminError> {
|
||||
match state
|
||||
.services
|
||||
.admin_sessions
|
||||
.validate(admin_session_cookie(headers).as_deref())
|
||||
.await
|
||||
{
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(AdminError::admin_session_required()),
|
||||
Err(_) => Err(AdminError::internal("Failed to validate admin session")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 要求请求携带有效管理员凭证:会话 Cookie 或管理员 API Key。
|
||||
pub(crate) async fn require_admin_auth(
|
||||
state: &AppState,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<(), AdminError> {
|
||||
if let Some(api_key) = admin_api_key_header(headers) {
|
||||
return match state.services.settings.verify_admin_api_key(&api_key).await {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(AdminError::invalid_admin_api_key()),
|
||||
Err(_) => Err(AdminError::internal("Failed to validate admin API key")),
|
||||
};
|
||||
}
|
||||
|
||||
require_admin_session(state, headers).await
|
||||
}
|
||||
|
||||
pub(super) fn admin_session_cookie(headers: &HeaderMap) -> Option<String> {
|
||||
let cookie = headers.get("cookie")?.to_str().ok()?;
|
||||
cookie.split(';').find_map(|part| {
|
||||
let (name, value) = part.trim().split_once('=')?;
|
||||
(name == ADMIN_SESSION_COOKIE).then(|| value.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
fn admin_api_key_header(headers: &HeaderMap) -> Option<String> {
|
||||
let value = headers.get("x-api-key")?.to_str().ok()?.trim();
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
}
|
||||
@@ -6,19 +6,16 @@ use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::{
|
||||
admin::auth::session::AdminAuth,
|
||||
admin::response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
config::{
|
||||
schema::AppConfig,
|
||||
settings::{AdminSettingsPatch, RuntimeSettingsError, SettingsServiceError},
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
session::AdminAuth,
|
||||
},
|
||||
runtime::state::AppState,
|
||||
upstream::models::config::ModelConfig,
|
||||
settings::{SettingsError, SettingsPatch, SettingsSnapshot, SettingsValidationError},
|
||||
};
|
||||
|
||||
const ALLOWED_SETTINGS_KEYS: [&str; 7] = [
|
||||
const ALLOWED_SETTINGS_KEYS: [&str; 6] = [
|
||||
"modelAliases",
|
||||
"modelAccountRoutes",
|
||||
"refreshMarginSeconds",
|
||||
"refreshConcurrency",
|
||||
"maxConcurrentPerAccount",
|
||||
@@ -30,7 +27,6 @@ const ALLOWED_SETTINGS_KEYS: [&str; 7] = [
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AdminSettingsData {
|
||||
model_aliases: BTreeMap<String, String>,
|
||||
model_account_routes: BTreeMap<String, Vec<String>>,
|
||||
refresh_margin_seconds: u64,
|
||||
refresh_concurrency: u32,
|
||||
max_concurrent_per_account: usize,
|
||||
@@ -39,15 +35,14 @@ struct AdminSettingsData {
|
||||
}
|
||||
|
||||
impl AdminSettingsData {
|
||||
fn from_config(config: &AppConfig) -> Self {
|
||||
fn from_settings(settings: &SettingsSnapshot) -> Self {
|
||||
Self {
|
||||
model_aliases: config.model_aliases.clone(),
|
||||
model_account_routes: config.model_account_routes.clone(),
|
||||
refresh_margin_seconds: config.auth.refresh_margin_seconds,
|
||||
refresh_concurrency: config.auth.refresh_concurrency,
|
||||
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
||||
request_interval_ms: config.auth.request_interval_ms,
|
||||
rotation_strategy: config.auth.rotation_strategy.clone(),
|
||||
model_aliases: settings.model_aliases.clone(),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: settings.refresh_concurrency,
|
||||
max_concurrent_per_account: settings.max_concurrent_per_account,
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
rotation_strategy: settings.rotation_strategy.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,7 +55,7 @@ pub(crate) async fn settings(
|
||||
let config = state.services.settings.current();
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminSettingsData::from_config(&config)),
|
||||
AdminEnvelope::ok(AdminSettingsData::from_settings(&config)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -77,13 +72,10 @@ pub(crate) async fn update_settings(
|
||||
.update(patch)
|
||||
.await
|
||||
.map_err(settings_service_error)?;
|
||||
state.services.models.update_config(ModelConfig {
|
||||
model_aliases: config.model_aliases.clone(),
|
||||
});
|
||||
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminSettingsData::from_config(&config)),
|
||||
AdminEnvelope::ok(AdminSettingsData::from_settings(&config)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -141,7 +133,7 @@ pub(crate) async fn delete_admin_api_key(
|
||||
))
|
||||
}
|
||||
|
||||
fn parse_settings_patch(body: &[u8]) -> Result<AdminSettingsPatch, AdminError> {
|
||||
fn parse_settings_patch(body: &[u8]) -> Result<SettingsPatch, AdminError> {
|
||||
let value = serde_json::from_slice::<Value>(body)
|
||||
.map_err(|_| AdminError::malformed_json("Malformed JSON body"))?;
|
||||
let object = value
|
||||
@@ -160,15 +152,12 @@ fn parse_settings_patch(body: &[u8]) -> Result<AdminSettingsPatch, AdminError> {
|
||||
serde_json::from_value(value).map_err(|_| AdminError::bad_request("Invalid settings payload"))
|
||||
}
|
||||
|
||||
fn settings_service_error(error: RuntimeSettingsError) -> AdminError {
|
||||
fn settings_service_error(error: SettingsError) -> AdminError {
|
||||
match error {
|
||||
RuntimeSettingsError::InvalidField(SettingsServiceError::InvalidField {
|
||||
field,
|
||||
message,
|
||||
}) => AdminError::bad_request(format!("Invalid setting {field}: {message}")),
|
||||
RuntimeSettingsError::Database(_)
|
||||
| RuntimeSettingsError::Json(_)
|
||||
| RuntimeSettingsError::StoredField { .. } => {
|
||||
SettingsError::InvalidField(SettingsValidationError::InvalidField { field, message }) => {
|
||||
AdminError::bad_request(format!("Invalid setting {field}: {message}"))
|
||||
}
|
||||
SettingsError::Database(_) | SettingsError::Json(_) | SettingsError::StoredField { .. } => {
|
||||
AdminError::settings_persist_failed("Failed to persist settings")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//! 管理端系统版本、自更新与重启 HTTP 处理器。
|
||||
|
||||
use std::{convert::Infallible, time::Duration};
|
||||
|
||||
use axum::{
|
||||
extract::{Json, Query, State},
|
||||
http::StatusCode,
|
||||
response::{
|
||||
IntoResponse,
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
},
|
||||
};
|
||||
use futures::Stream;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{AdminEnvelope, AdminError, AdminResponse},
|
||||
session::AdminAuth,
|
||||
},
|
||||
update::{service::RestartAction, state::operation_id, types::UpdateError},
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UpdateDetailQuery {
|
||||
refresh: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UpdateRequest {
|
||||
target_version: String,
|
||||
}
|
||||
|
||||
pub(crate) async fn version(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(state.services.system_update.version_data().await),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn update_detail(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<UpdateDetailQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(
|
||||
state
|
||||
.services
|
||||
.system_update
|
||||
.update_detail(query.refresh.unwrap_or(false))
|
||||
.await,
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn update_event_stream(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, AdminError> {
|
||||
let receiver = state.services.system_update.subscribe_update_events();
|
||||
let stream = futures::stream::unfold(
|
||||
(receiver, false),
|
||||
|(mut receiver, close_after_send)| async move {
|
||||
if close_after_send {
|
||||
return None;
|
||||
}
|
||||
loop {
|
||||
match receiver.recv().await {
|
||||
Ok(message) => {
|
||||
let close_after_send = message.is_terminal();
|
||||
let id = message.id().to_string();
|
||||
let data =
|
||||
serde_json::to_string(&message).unwrap_or_else(|_| "{}".to_string());
|
||||
let event = Event::default().event("update").id(id).data(data);
|
||||
return Some((Ok(event), (receiver, close_after_send)));
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => return None,
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
|
||||
}
|
||||
|
||||
pub(crate) async fn perform_update(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
payload: Option<Json<UpdateRequest>>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let target = payload.map(|Json(payload)| payload.target_version);
|
||||
let result = state
|
||||
.services
|
||||
.system_update
|
||||
.perform_update(target)
|
||||
.await
|
||||
.map_err(update_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(result),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn update_status(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let status = state
|
||||
.services
|
||||
.system_update
|
||||
.update_status()
|
||||
.map_err(update_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(status),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn rollback(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let operation_id = state
|
||||
.services
|
||||
.system_update
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(update_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(serde_json::json!({
|
||||
"message": "回滚完成,请重启服务。",
|
||||
"needRestart": true,
|
||||
"operationId": operation_id,
|
||||
})),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn restart(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let plan = state
|
||||
.services
|
||||
.system_update
|
||||
.restart_plan()
|
||||
.map_err(update_error)?;
|
||||
let message = plan.message;
|
||||
let process_control = state.services.process_control.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
match plan.action {
|
||||
RestartAction::Exec(executable_path) => {
|
||||
process_control.request_restart(executable_path);
|
||||
}
|
||||
RestartAction::Shutdown => process_control.request_shutdown(),
|
||||
}
|
||||
});
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(serde_json::json!({
|
||||
"message": message,
|
||||
"operationId": operation_id("restart"),
|
||||
})),
|
||||
))
|
||||
}
|
||||
|
||||
fn update_error(error: UpdateError) -> AdminError {
|
||||
match error {
|
||||
UpdateError::BadRequest(message) => AdminError::bad_request(message),
|
||||
UpdateError::Conflict(message) => AdminError::conflict(message),
|
||||
UpdateError::BadGateway(message) => AdminError::bad_gateway(message),
|
||||
UpdateError::Internal(message) => AdminError::internal(message),
|
||||
}
|
||||
}
|
||||
+130
-216
@@ -11,43 +11,37 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
admin::monitoring::{
|
||||
usage_record_model::{
|
||||
metadata_nonnegative_i64, metadata_string, UsageRecord, UsageRecordLevel,
|
||||
},
|
||||
usage_record_service::{
|
||||
usage_record_cost_details, AdminUsageRecordError, AdminUsageRecordFilter,
|
||||
},
|
||||
usage_record_store::{
|
||||
UsageRecordBreakdown, UsageRecordEndpointSource, UsageRecordModelSource,
|
||||
UsageRecordSummary, UsageRecordTrendPoint,
|
||||
},
|
||||
},
|
||||
admin::{
|
||||
auth::session::AdminAuth,
|
||||
api::AppState,
|
||||
api::admin::{
|
||||
response::{AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse},
|
||||
session::AdminAuth,
|
||||
},
|
||||
infra::{
|
||||
format::{
|
||||
format_compact_number, format_cost, format_duration_ms, format_duration_ms_f64,
|
||||
format_multiplier, format_number, format_percent, format_rate, format_token_price,
|
||||
format_tokens,
|
||||
format_billing_amount, format_compact_number, format_duration_ms,
|
||||
format_duration_ms_f64, format_multiplier, format_number, format_percent,
|
||||
format_token_price, format_tokens,
|
||||
},
|
||||
json::{clamp_limit, clamp_page, NumberedPage, Page},
|
||||
json::{NumberedPage, clamp_limit, clamp_page},
|
||||
time::china_datetime,
|
||||
},
|
||||
runtime::state::AppState,
|
||||
telemetry::{
|
||||
usage::query::{
|
||||
UsageQueryError, UsageQueryFilter, UsageRecordBreakdown, UsageRecordModelSource,
|
||||
UsageRecordSummary, UsageRecordTrendPoint, usage_record_billing,
|
||||
},
|
||||
usage::types::{UsageRecord, metadata_string},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UsageRecordsQuery {
|
||||
cursor: Option<String>,
|
||||
limit: Option<u32>,
|
||||
page: Option<u32>,
|
||||
page_size: Option<u32>,
|
||||
kind: Option<String>,
|
||||
level: Option<String>,
|
||||
client_api_key_id: Option<String>,
|
||||
provider: Option<String>,
|
||||
request_id: Option<String>,
|
||||
account_id: Option<String>,
|
||||
route: Option<String>,
|
||||
@@ -55,8 +49,6 @@ pub(crate) struct UsageRecordsQuery {
|
||||
status_code: Option<i64>,
|
||||
transport: Option<String>,
|
||||
attempt_index: Option<i64>,
|
||||
upstream_status_code: Option<i64>,
|
||||
failure_class: Option<String>,
|
||||
response_id: Option<String>,
|
||||
upstream_request_id: Option<String>,
|
||||
search: Option<String>,
|
||||
@@ -96,8 +88,11 @@ pub(crate) struct UsageRecordData {
|
||||
client_ip: Option<String>,
|
||||
user_agent: Option<String>,
|
||||
reasoning_effort: Option<String>,
|
||||
compact: bool,
|
||||
request_kind: Option<String>,
|
||||
subagent_kind: Option<String>,
|
||||
token_details: UsageRecordTokenDetailsData,
|
||||
cost_details: Option<UsageRecordCostDetailsData>,
|
||||
billing: Option<UsageRecordBillingData>,
|
||||
first_token_latency_ms: Option<i64>,
|
||||
first_token_latency_ms_display: String,
|
||||
latency_ms_display: String,
|
||||
@@ -120,23 +115,23 @@ pub(crate) struct UsageRecordTokenDetailsData {
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct UsageRecordCostDetailsData {
|
||||
input_cost: f64,
|
||||
output_cost: f64,
|
||||
cache_read_cost: f64,
|
||||
total_cost: f64,
|
||||
original_cost: f64,
|
||||
pub(crate) struct UsageRecordBillingData {
|
||||
input_amount: f64,
|
||||
output_amount: f64,
|
||||
cache_read_amount: f64,
|
||||
standard_amount: f64,
|
||||
total_amount: f64,
|
||||
input_price_per_mtoken: f64,
|
||||
output_price_per_mtoken: f64,
|
||||
cache_read_price_per_mtoken: f64,
|
||||
service_tier: Option<String>,
|
||||
service_tier_display: String,
|
||||
multiplier: f64,
|
||||
input_cost_display: String,
|
||||
output_cost_display: String,
|
||||
cache_read_cost_display: String,
|
||||
total_cost_display: String,
|
||||
original_cost_display: String,
|
||||
input_amount_display: String,
|
||||
output_amount_display: String,
|
||||
cache_read_amount_display: String,
|
||||
standard_amount_display: String,
|
||||
total_amount_display: String,
|
||||
input_price_display: String,
|
||||
output_price_display: String,
|
||||
cache_read_price_display: String,
|
||||
@@ -147,8 +142,6 @@ pub(crate) struct UsageRecordCostDetailsData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UsageRecordSummaryData {
|
||||
total_requests: String,
|
||||
error_requests: String,
|
||||
error_rate: String,
|
||||
input_tokens: String,
|
||||
output_tokens: String,
|
||||
cached_tokens: String,
|
||||
@@ -169,9 +162,9 @@ struct UsageRecordBreakdownData {
|
||||
total_tokens_value: u64,
|
||||
total_tokens_total: String,
|
||||
total_tokens_total_value: u64,
|
||||
cost: String,
|
||||
actual_cost: String,
|
||||
account_cost: String,
|
||||
standard_billing_amount: String,
|
||||
actual_billing_amount: String,
|
||||
account_billing_amount: String,
|
||||
average_latency_ms: String,
|
||||
}
|
||||
|
||||
@@ -183,16 +176,14 @@ struct UsageRecordTrendPointData {
|
||||
input_tokens_value: u64,
|
||||
output_tokens: String,
|
||||
output_tokens_value: u64,
|
||||
cache_creation_tokens: String,
|
||||
cache_creation_tokens_value: u64,
|
||||
cached_tokens: String,
|
||||
cached_tokens_value: u64,
|
||||
total_tokens: String,
|
||||
total_tokens_value: u64,
|
||||
cache_hit_rate: String,
|
||||
cache_hit_rate_value: f64,
|
||||
cost: String,
|
||||
actual_cost: String,
|
||||
standard_billing_amount: String,
|
||||
actual_billing_amount: String,
|
||||
average_latency_ms: String,
|
||||
average_latency_ms_value: Option<f64>,
|
||||
}
|
||||
@@ -203,45 +194,14 @@ pub(crate) async fn usage_records(
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<UsageRecordsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let limit = clamp_limit(query.page_size.or(query.limit).unwrap_or(50));
|
||||
let page = query.page;
|
||||
let use_numbered_page = page.is_some() || query.page_size.is_some();
|
||||
let cursor = query.cursor.clone();
|
||||
let page = clamp_page(query.page.unwrap_or(1));
|
||||
let page_size = clamp_limit(query.page_size.unwrap_or(50));
|
||||
let filter = filter_from_query(query)?;
|
||||
|
||||
if use_numbered_page {
|
||||
return match state
|
||||
.services
|
||||
.usage_records
|
||||
.list_page(clamp_page(page.unwrap_or(1)), limit, filter)
|
||||
.await
|
||||
{
|
||||
Ok(page) => {
|
||||
let account_emails = state
|
||||
.services
|
||||
.usage_records
|
||||
.account_email_map(&page.items)
|
||||
.await
|
||||
.map_err(|error| log_error(&error))?;
|
||||
let page = NumberedPage {
|
||||
items: usage_record_items(page.items, &account_emails),
|
||||
total: page.total,
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
};
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminPageEnvelope::numbered(page),
|
||||
))
|
||||
}
|
||||
Err(error) => Err(log_error(&error)),
|
||||
};
|
||||
}
|
||||
|
||||
match state
|
||||
.services
|
||||
.usage_records
|
||||
.list(cursor, limit, filter)
|
||||
.list_page(page, page_size, filter)
|
||||
.await
|
||||
{
|
||||
Ok(page) => {
|
||||
@@ -251,13 +211,15 @@ pub(crate) async fn usage_records(
|
||||
.account_email_map(&page.items)
|
||||
.await
|
||||
.map_err(|error| log_error(&error))?;
|
||||
let page = Page {
|
||||
let page = NumberedPage {
|
||||
items: usage_record_items(page.items, &account_emails),
|
||||
next_cursor: page.next_cursor,
|
||||
total: page.total,
|
||||
page: page.page,
|
||||
page_size: page.page_size,
|
||||
};
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminPageEnvelope::ok(page, limit),
|
||||
AdminPageEnvelope::ok(page),
|
||||
))
|
||||
}
|
||||
Err(error) => Err(log_error(&error)),
|
||||
@@ -306,14 +268,13 @@ pub(crate) async fn usage_records_model_distribution(
|
||||
pub(crate) async fn usage_records_endpoint_distribution(
|
||||
State(state): State<AppState>,
|
||||
_auth: AdminAuth,
|
||||
Query(query): Query<UsageRecordDistributionQuery>,
|
||||
Query(query): Query<UsageRecordsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError> {
|
||||
let source = endpoint_source_from_query(query.source)?;
|
||||
let filter = filter_from_query(query.records)?;
|
||||
let filter = filter_from_query(query)?;
|
||||
match state
|
||||
.services
|
||||
.usage_records
|
||||
.endpoint_distribution(filter, source)
|
||||
.endpoint_distribution(filter)
|
||||
.await
|
||||
{
|
||||
Ok(items) => Ok(AdminResponse::new(
|
||||
@@ -401,10 +362,11 @@ pub(crate) async fn clear_usage_records(
|
||||
}
|
||||
}
|
||||
|
||||
fn filter_from_query(query: UsageRecordsQuery) -> Result<AdminUsageRecordFilter, AdminError> {
|
||||
Ok(AdminUsageRecordFilter {
|
||||
fn filter_from_query(query: UsageRecordsQuery) -> Result<UsageQueryFilter, AdminError> {
|
||||
Ok(UsageQueryFilter {
|
||||
kind: non_empty(query.kind),
|
||||
level: level_from_query(query.level).map_err(AdminError::bad_request)?,
|
||||
client_api_key_id: non_empty(query.client_api_key_id),
|
||||
provider: non_empty(query.provider),
|
||||
request_id: non_empty(query.request_id),
|
||||
account_id: non_empty(query.account_id),
|
||||
route: non_empty(query.route),
|
||||
@@ -412,8 +374,6 @@ fn filter_from_query(query: UsageRecordsQuery) -> Result<AdminUsageRecordFilter,
|
||||
status_code: query.status_code,
|
||||
transport: non_empty(query.transport),
|
||||
attempt_index: query.attempt_index,
|
||||
upstream_status_code: query.upstream_status_code,
|
||||
failure_class: non_empty(query.failure_class),
|
||||
response_id: non_empty(query.response_id),
|
||||
upstream_request_id: non_empty(query.upstream_request_id),
|
||||
search: non_empty(query.search),
|
||||
@@ -433,19 +393,6 @@ fn model_source_from_query(value: Option<String>) -> Result<UsageRecordModelSour
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_source_from_query(
|
||||
value: Option<String>,
|
||||
) -> Result<UsageRecordEndpointSource, AdminError> {
|
||||
match non_empty(value).as_deref().unwrap_or("inbound") {
|
||||
"inbound" => Ok(UsageRecordEndpointSource::Inbound),
|
||||
"upstream" => Ok(UsageRecordEndpointSource::Upstream),
|
||||
"path" => Ok(UsageRecordEndpointSource::Path),
|
||||
_ => Err(AdminError::invalid_endpoint_source(
|
||||
"Invalid endpoint distribution source",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_datetime(
|
||||
value: Option<String>,
|
||||
) -> Result<Option<chrono::DateTime<chrono::Utc>>, AdminError> {
|
||||
@@ -457,29 +404,12 @@ fn optional_datetime(
|
||||
.map_err(|_| AdminError::invalid_time_range("Invalid time range"))
|
||||
}
|
||||
|
||||
fn log_error(error: &AdminUsageRecordError) -> AdminError {
|
||||
fn log_error(error: &UsageQueryError) -> AdminError {
|
||||
match error {
|
||||
AdminUsageRecordError::List
|
||||
| AdminUsageRecordError::Get
|
||||
| AdminUsageRecordError::Clear
|
||||
| AdminUsageRecordError::Append
|
||||
| AdminUsageRecordError::Trim => AdminError::internal(error.to_string()),
|
||||
AdminUsageRecordError::Accounts => {
|
||||
AdminError::usage_record_accounts_failed(error.to_string())
|
||||
UsageQueryError::List | UsageQueryError::Get | UsageQueryError::Clear => {
|
||||
AdminError::internal(error.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn level_from_query(value: Option<String>) -> Result<Option<UsageRecordLevel>, String> {
|
||||
let Some(value) = non_empty(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
match value.as_str() {
|
||||
"debug" => Ok(Some(UsageRecordLevel::Debug)),
|
||||
"info" => Ok(Some(UsageRecordLevel::Info)),
|
||||
"warn" => Ok(Some(UsageRecordLevel::Warn)),
|
||||
"error" => Ok(Some(UsageRecordLevel::Error)),
|
||||
other => Err(format!("Unsupported log level: {other}")),
|
||||
UsageQueryError::Accounts => AdminError::usage_record_accounts_failed(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -503,10 +433,8 @@ fn usage_record_data(
|
||||
record: UsageRecord,
|
||||
account_emails: &HashMap<String, String>,
|
||||
) -> UsageRecordData {
|
||||
let account_email = record
|
||||
.account_id
|
||||
.as_deref()
|
||||
.and_then(|account_id| account_emails.get(account_id))
|
||||
let account_email = account_emails
|
||||
.get(&record.account_id)
|
||||
.cloned()
|
||||
.or_else(|| metadata_string(&record.metadata, &["accountEmail", "account_email"]));
|
||||
let (requested_model, upstream_model) = usage_record_models(&record);
|
||||
@@ -514,17 +442,16 @@ fn usage_record_data(
|
||||
let user_agent = metadata_string(&record.metadata, &["userAgent", "user_agent"]);
|
||||
let reasoning_effort =
|
||||
metadata_string(&record.metadata, &["reasoningEffort", "reasoning_effort"]);
|
||||
let token_details = usage_token_details(&record.metadata);
|
||||
let cost_details = usage_cost_details(&record, upstream_model.as_deref(), &token_details);
|
||||
let first_token_latency_ms = metadata_nonnegative_i64(
|
||||
&record.metadata,
|
||||
&[
|
||||
"firstTokenMs",
|
||||
"first_token_ms",
|
||||
"firstTokenLatencyMs",
|
||||
"first_token_latency_ms",
|
||||
],
|
||||
);
|
||||
let compact = record
|
||||
.metadata
|
||||
.get("compact")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let request_kind = metadata_string(&record.metadata, &["requestKind", "request_kind"]);
|
||||
let subagent_kind = metadata_string(&record.metadata, &["subagentKind", "subagent_kind"]);
|
||||
let token_details = usage_token_details(&record);
|
||||
let billing = usage_billing(&record, upstream_model.as_deref(), &token_details);
|
||||
let first_token_latency_ms = record.first_token_ms;
|
||||
let first_token_latency_ms_display = format_duration_ms(first_token_latency_ms);
|
||||
let latency_ms_display = format_duration_ms(record.latency_ms);
|
||||
|
||||
@@ -536,8 +463,11 @@ fn usage_record_data(
|
||||
client_ip,
|
||||
user_agent,
|
||||
reasoning_effort,
|
||||
compact,
|
||||
request_kind,
|
||||
subagent_kind,
|
||||
token_details,
|
||||
cost_details,
|
||||
billing,
|
||||
first_token_latency_ms,
|
||||
first_token_latency_ms_display,
|
||||
latency_ms_display,
|
||||
@@ -545,28 +475,39 @@ fn usage_record_data(
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_record_models(record: &UsageRecord) -> (Option<String>, Option<String>) {
|
||||
let stored_model = record
|
||||
.model
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|model| !model.is_empty());
|
||||
let requested_model = metadata_string(&record.metadata, &["requestedModel"])
|
||||
.or_else(|| stored_model.map(ToString::to_string));
|
||||
let upstream_model = metadata_string(&record.metadata, &["upstreamModel"])
|
||||
pub(crate) fn usage_record_models(record: &UsageRecord) -> (Option<String>, Option<String>) {
|
||||
let stored_model = record.model.trim();
|
||||
let requested_model = record
|
||||
.requested_model
|
||||
.clone()
|
||||
.or_else(|| (!stored_model.is_empty()).then(|| stored_model.to_string()));
|
||||
let upstream_model = record
|
||||
.upstream_model
|
||||
.clone()
|
||||
.or_else(|| requested_model.clone())
|
||||
.or_else(|| stored_model.map(ToString::to_string));
|
||||
.or_else(|| (!stored_model.is_empty()).then(|| stored_model.to_string()));
|
||||
|
||||
(requested_model, upstream_model)
|
||||
}
|
||||
|
||||
fn usage_token_details(metadata: &Value) -> UsageRecordTokenDetailsData {
|
||||
let input_tokens = metadata_usage_u64(metadata, "inputTokens");
|
||||
let output_tokens = metadata_usage_u64(metadata, "outputTokens");
|
||||
let cached_tokens = metadata_usage_u64(metadata, "cachedTokens");
|
||||
let reasoning_tokens = metadata_usage_u64(metadata, "reasoningTokens");
|
||||
let total_tokens =
|
||||
metadata_usage_u64(metadata, "totalTokens").max(input_tokens.saturating_add(output_tokens));
|
||||
pub(crate) fn usage_token_details(record: &UsageRecord) -> UsageRecordTokenDetailsData {
|
||||
let input_tokens: u64 = record
|
||||
.input_tokens
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.unwrap_or(0);
|
||||
let output_tokens: u64 = record
|
||||
.output_tokens
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.unwrap_or(0);
|
||||
let cached_tokens: u64 = record
|
||||
.cached_tokens
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.unwrap_or(0);
|
||||
let reasoning_tokens: u64 = record
|
||||
.reasoning_tokens
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.unwrap_or(0);
|
||||
let total_tokens = input_tokens.saturating_add(output_tokens);
|
||||
|
||||
UsageRecordTokenDetailsData {
|
||||
input_tokens,
|
||||
@@ -582,12 +523,12 @@ fn usage_token_details(metadata: &Value) -> UsageRecordTokenDetailsData {
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_cost_details(
|
||||
pub(crate) fn usage_billing(
|
||||
record: &UsageRecord,
|
||||
upstream_model: Option<&str>,
|
||||
tokens: &UsageRecordTokenDetailsData,
|
||||
) -> Option<UsageRecordCostDetailsData> {
|
||||
let cost = usage_record_cost_details(
|
||||
) -> Option<UsageRecordBillingData> {
|
||||
let billing = usage_record_billing(
|
||||
record,
|
||||
upstream_model,
|
||||
tokens.input_tokens,
|
||||
@@ -595,56 +536,34 @@ fn usage_cost_details(
|
||||
tokens.cached_tokens,
|
||||
)?;
|
||||
|
||||
Some(UsageRecordCostDetailsData {
|
||||
input_cost: cost.input_cost,
|
||||
output_cost: cost.output_cost,
|
||||
cache_read_cost: cost.cache_read_cost,
|
||||
total_cost: cost.total_cost,
|
||||
original_cost: cost.original_cost,
|
||||
input_price_per_mtoken: cost.input_price_per_mtoken,
|
||||
output_price_per_mtoken: cost.output_price_per_mtoken,
|
||||
cache_read_price_per_mtoken: cost.cache_read_price_per_mtoken,
|
||||
service_tier: cost.service_tier,
|
||||
service_tier_display: cost.service_tier_display,
|
||||
multiplier: cost.multiplier,
|
||||
input_cost_display: format_cost(cost.input_cost),
|
||||
output_cost_display: format_cost(cost.output_cost),
|
||||
cache_read_cost_display: format_cost(cost.cache_read_cost),
|
||||
total_cost_display: format_cost(cost.total_cost),
|
||||
original_cost_display: format_cost(cost.original_cost),
|
||||
input_price_display: format_token_price(cost.input_price_per_mtoken),
|
||||
output_price_display: format_token_price(cost.output_price_per_mtoken),
|
||||
cache_read_price_display: format_token_price(cost.cache_read_price_per_mtoken),
|
||||
multiplier_display: format_multiplier(cost.multiplier),
|
||||
Some(UsageRecordBillingData {
|
||||
input_amount: billing.input_amount,
|
||||
output_amount: billing.output_amount,
|
||||
cache_read_amount: billing.cache_read_amount,
|
||||
standard_amount: billing.standard_amount,
|
||||
total_amount: billing.total_amount,
|
||||
input_price_per_mtoken: billing.input_price_per_mtoken,
|
||||
output_price_per_mtoken: billing.output_price_per_mtoken,
|
||||
cache_read_price_per_mtoken: billing.cache_read_price_per_mtoken,
|
||||
service_tier: billing.service_tier,
|
||||
service_tier_display: billing.service_tier_display,
|
||||
multiplier: billing.multiplier,
|
||||
input_amount_display: format_billing_amount(billing.input_amount),
|
||||
output_amount_display: format_billing_amount(billing.output_amount),
|
||||
cache_read_amount_display: format_billing_amount(billing.cache_read_amount),
|
||||
standard_amount_display: format_billing_amount(billing.standard_amount),
|
||||
total_amount_display: format_billing_amount(billing.total_amount),
|
||||
input_price_display: format_token_price(billing.input_price_per_mtoken),
|
||||
output_price_display: format_token_price(billing.output_price_per_mtoken),
|
||||
cache_read_price_display: format_token_price(billing.cache_read_price_per_mtoken),
|
||||
multiplier_display: format_multiplier(billing.multiplier),
|
||||
})
|
||||
}
|
||||
|
||||
fn metadata_usage_u64(metadata: &Value, key: &str) -> u64 {
|
||||
metadata
|
||||
.get("usage")
|
||||
.and_then(|usage| metadata_u64_at(usage, key))
|
||||
.or_else(|| metadata_u64_at(metadata, key))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn metadata_u64_at(value: &Value, key: &str) -> Option<u64> {
|
||||
value
|
||||
.get(key)
|
||||
.and_then(|value| value.as_u64().or_else(|| value.as_i64()?.try_into().ok()))
|
||||
}
|
||||
|
||||
impl From<UsageRecordSummary> for UsageRecordSummaryData {
|
||||
fn from(summary: UsageRecordSummary) -> Self {
|
||||
Self {
|
||||
total_requests: format_compact_number(summary.total_requests),
|
||||
error_requests: format_compact_number(summary.error_requests),
|
||||
error_rate: if summary.total_requests > 0 {
|
||||
format_rate(Some(
|
||||
summary.error_requests as f64 / summary.total_requests as f64,
|
||||
))
|
||||
} else {
|
||||
"—".to_string()
|
||||
},
|
||||
input_tokens: format_tokens(summary.input_tokens),
|
||||
output_tokens: format_tokens(summary.output_tokens),
|
||||
cached_tokens: format_tokens(summary.cached_tokens),
|
||||
@@ -656,7 +575,7 @@ impl From<UsageRecordSummary> for UsageRecordSummaryData {
|
||||
|
||||
impl From<UsageRecordBreakdown> for UsageRecordBreakdownData {
|
||||
fn from(item: UsageRecordBreakdown) -> Self {
|
||||
UsageRecordBreakdownData::from_item(item, 0)
|
||||
Self::from_item(item, 0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -673,9 +592,9 @@ impl UsageRecordBreakdownData {
|
||||
total_tokens_value: item.total_tokens,
|
||||
total_tokens_total: format_tokens(total_tokens_total),
|
||||
total_tokens_total_value: total_tokens_total,
|
||||
cost: format_cost(item.cost),
|
||||
actual_cost: format_cost(item.actual_cost),
|
||||
account_cost: format_cost(item.account_cost),
|
||||
standard_billing_amount: format_billing_amount(item.standard_billing_amount),
|
||||
actual_billing_amount: format_billing_amount(item.actual_billing_amount),
|
||||
account_billing_amount: format_billing_amount(item.account_billing_amount),
|
||||
average_latency_ms: format_duration_ms_f64(item.average_latency_ms),
|
||||
}
|
||||
}
|
||||
@@ -694,10 +613,7 @@ fn usage_record_breakdown_items(items: Vec<UsageRecordBreakdown>) -> Vec<UsageRe
|
||||
|
||||
impl From<UsageRecordTrendPoint> for UsageRecordTrendPointData {
|
||||
fn from(point: UsageRecordTrendPoint) -> Self {
|
||||
let prompt_tokens = point
|
||||
.input_tokens
|
||||
.saturating_add(point.cache_creation_tokens)
|
||||
.saturating_add(point.cached_tokens);
|
||||
let prompt_tokens = point.input_tokens.saturating_add(point.cached_tokens);
|
||||
let cache_hit_rate_value = if prompt_tokens > 0 {
|
||||
(point.cached_tokens as f64 / prompt_tokens as f64 * 100.0).round()
|
||||
} else {
|
||||
@@ -709,16 +625,14 @@ impl From<UsageRecordTrendPoint> for UsageRecordTrendPointData {
|
||||
input_tokens_value: point.input_tokens,
|
||||
output_tokens: format_tokens(point.output_tokens),
|
||||
output_tokens_value: point.output_tokens,
|
||||
cache_creation_tokens: format_tokens(point.cache_creation_tokens),
|
||||
cache_creation_tokens_value: point.cache_creation_tokens,
|
||||
cached_tokens: format_tokens(point.cached_tokens),
|
||||
cached_tokens_value: point.cached_tokens,
|
||||
total_tokens: format_tokens(point.total_tokens),
|
||||
total_tokens_value: point.total_tokens,
|
||||
cache_hit_rate: format_percent(cache_hit_rate_value),
|
||||
cache_hit_rate_value,
|
||||
cost: format_cost(point.cost),
|
||||
actual_cost: format_cost(point.actual_cost),
|
||||
standard_billing_amount: format_billing_amount(point.standard_billing_amount),
|
||||
actual_billing_amount: format_billing_amount(point.actual_billing_amount),
|
||||
average_latency_ms: format_duration_ms_f64(point.average_latency_ms),
|
||||
average_latency_ms_value: point.average_latency_ms,
|
||||
}
|
||||
@@ -4,10 +4,10 @@ use std::{
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json, Router,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tower::service_fn;
|
||||
@@ -2,9 +2,9 @@
|
||||
//!
|
||||
//! 提供 Bearer token 提取与客户端 API key 校验。
|
||||
|
||||
use axum::http::{header::AUTHORIZATION, HeaderMap};
|
||||
use axum::http::{HeaderMap, header::AUTHORIZATION};
|
||||
|
||||
use crate::runtime::state::AppState;
|
||||
use crate::api::AppState;
|
||||
|
||||
/// client API key 鉴权失败原因。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -56,11 +56,16 @@ pub fn bearer_client_api_key(headers: &HeaderMap) -> Result<&str, ClientApiKeyAu
|
||||
|
||||
/// 校验客户端 API key。
|
||||
pub async fn authorize_client_api_key(state: &AppState, headers: &HeaderMap) -> bool {
|
||||
authorized_client_api_key_id(state, headers).await.is_some()
|
||||
}
|
||||
|
||||
/// 校验客户端 API key,成功时返回用于事实归因的稳定 ID。
|
||||
pub async fn authorized_client_api_key_id(state: &AppState, headers: &HeaderMap) -> Option<String> {
|
||||
match authorize_client_api_key_result(state, headers).await {
|
||||
Ok(()) => true,
|
||||
Ok(key_id) => Some(key_id),
|
||||
Err(error) => {
|
||||
log_client_api_key_auth_failure(&error);
|
||||
false
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,7 +74,7 @@ pub async fn authorize_client_api_key(state: &AppState, headers: &HeaderMap) ->
|
||||
pub async fn authorize_client_api_key_result(
|
||||
state: &AppState,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<(), ClientApiKeyAuthError> {
|
||||
) -> Result<String, ClientApiKeyAuthError> {
|
||||
let api_key = bearer_client_api_key(headers)?;
|
||||
let verified = state
|
||||
.services
|
||||
@@ -79,9 +84,7 @@ pub async fn authorize_client_api_key_result(
|
||||
.map_err(|error| ClientApiKeyAuthError::StoreUnavailable {
|
||||
message: error.to_string(),
|
||||
})?;
|
||||
verified
|
||||
.then_some(())
|
||||
.ok_or(ClientApiKeyAuthError::InvalidKey)
|
||||
verified.ok_or(ClientApiKeyAuthError::InvalidKey)
|
||||
}
|
||||
|
||||
fn log_client_api_key_auth_failure(error: &ClientApiKeyAuthError) {
|
||||
@@ -94,7 +97,7 @@ fn log_client_api_key_auth_failure(error: &ClientApiKeyAuthError) {
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
tracing::info!(
|
||||
auth_failure = error.reason(),
|
||||
"client api key authorization failed"
|
||||
);
|
||||
@@ -1,15 +1,16 @@
|
||||
//! OpenAI 路由共享错误响应。
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::proxy::{
|
||||
dispatch::{errors::DispatchFailureClass, responses::errors::ResponseDispatchError},
|
||||
openai::responses::response_failed_sse_event as encode_response_failed_sse_event,
|
||||
use crate::{
|
||||
dispatch::errors::{DispatchFailureClass, ResponseDispatchError},
|
||||
upstream::openai::protocol::sse::response_failed_sse_event as encode_response_failed_sse_event,
|
||||
upstream::openai::transport::CodexClientError,
|
||||
};
|
||||
|
||||
const NO_ACTIVE_UPSTREAM_ACCOUNT_MESSAGE: &str = "No active upstream account is available";
|
||||
@@ -18,29 +19,26 @@ const NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE: &str =
|
||||
const UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE: &str = "Upstream Codex request failed";
|
||||
const INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE: &str = "Invalid upstream Codex response";
|
||||
const UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE: &str = "Upstream Codex response failed";
|
||||
const HISTORY_UNAVAILABLE_MESSAGE: &str = "Previous response context is unavailable. Start a new conversation or resend the complete input without previous_response_id.";
|
||||
const CONTINUATION_BUSY_MESSAGE: &str =
|
||||
"The previous response connection is busy. Retry the request.";
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum ResponseDispatchMessageStyle {
|
||||
enum ResponseDispatchMessageStyle {
|
||||
Standard,
|
||||
ResponsesStream,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum ResponseDispatchStatusMode {
|
||||
UpstreamFailureStatus,
|
||||
Client,
|
||||
struct DispatchHttpError {
|
||||
status: StatusCode,
|
||||
message: String,
|
||||
}
|
||||
|
||||
pub struct DispatchHttpError {
|
||||
pub status: StatusCode,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
pub struct OpenAiErrorDetails {
|
||||
pub status: StatusCode,
|
||||
pub message: String,
|
||||
pub error_type: &'static str,
|
||||
pub code: &'static str,
|
||||
struct OpenAiErrorDetails {
|
||||
status: StatusCode,
|
||||
message: String,
|
||||
error_type: &'static str,
|
||||
code: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -49,9 +47,9 @@ struct OpenAiErrorKind {
|
||||
code: &'static str,
|
||||
}
|
||||
|
||||
const UPSTREAM_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
const NO_AVAILABLE_ACCOUNTS_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "upstream_unavailable",
|
||||
code: "no_available_accounts",
|
||||
};
|
||||
const UPSTREAM_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
@@ -73,6 +71,14 @@ const MODEL_NOT_FOUND_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "invalid_request_error",
|
||||
code: "model_not_found",
|
||||
};
|
||||
const HISTORY_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "invalid_request_error",
|
||||
code: "previous_response_unavailable",
|
||||
};
|
||||
const CONTINUATION_BUSY_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "continuation_connection_busy",
|
||||
};
|
||||
const INVALID_UPSTREAM_RESPONSE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
@@ -81,6 +87,10 @@ const CODEX_API_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "codex_api_error",
|
||||
};
|
||||
const CODEX_CLIENT_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "invalid_request_error",
|
||||
code: "codex_client_error",
|
||||
};
|
||||
|
||||
/// OpenAI 兼容错误响应。
|
||||
pub fn openai_error_response(
|
||||
@@ -102,62 +112,114 @@ pub fn openai_error_response(
|
||||
}
|
||||
|
||||
pub fn responses_dispatch_error_response(error: ResponseDispatchError) -> Response {
|
||||
match error {
|
||||
ResponseDispatchError::NoActiveAccount => {
|
||||
responses_no_available_accounts_response().into_response()
|
||||
}
|
||||
error => response_dispatch_openai_error_response(&error),
|
||||
}
|
||||
responses_dispatch_error_response_ref(&error)
|
||||
}
|
||||
|
||||
pub fn responses_compact_dispatch_error_response(error: ResponseDispatchError) -> Response {
|
||||
pub fn responses_dispatch_error_response_ref(error: &ResponseDispatchError) -> Response {
|
||||
if let ResponseDispatchError::Upstream(CodexClientError::Upstream { status, body, .. }) = error
|
||||
&& status.is_client_error()
|
||||
&& let Ok(body) = serde_json::from_str::<Value>(body)
|
||||
{
|
||||
return (*status, Json(body)).into_response();
|
||||
}
|
||||
|
||||
match error {
|
||||
ResponseDispatchError::NoActiveAccount => {
|
||||
responses_no_available_accounts_response().into_response()
|
||||
}
|
||||
error => {
|
||||
let error = response_dispatch_http_error(
|
||||
&error,
|
||||
ResponseDispatchStatusMode::Client,
|
||||
ResponseDispatchMessageStyle::Standard,
|
||||
);
|
||||
responses_error_response(error.status, &error.message).into_response()
|
||||
}
|
||||
error => response_dispatch_openai_error_response(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn responses_stream_dispatch_failed_sse_event(error: &ResponseDispatchError) -> String {
|
||||
let http_error = response_dispatch_http_error(
|
||||
error,
|
||||
ResponseDispatchStatusMode::UpstreamFailureStatus,
|
||||
ResponseDispatchMessageStyle::ResponsesStream,
|
||||
);
|
||||
let http_error =
|
||||
response_dispatch_http_error(error, ResponseDispatchMessageStyle::ResponsesStream);
|
||||
let kind = response_dispatch_error_kind(error);
|
||||
responses_failed_sse_event(kind, &http_error.message)
|
||||
}
|
||||
|
||||
pub(super) fn responses_websocket_dispatch_error_event(
|
||||
error: &ResponseDispatchError,
|
||||
request_id: &str,
|
||||
) -> String {
|
||||
if let Some(event) = upstream_client_websocket_error_event(error, request_id) {
|
||||
return event;
|
||||
}
|
||||
let http_error =
|
||||
response_dispatch_http_error(error, ResponseDispatchMessageStyle::ResponsesStream);
|
||||
let kind = response_dispatch_error_kind(error);
|
||||
responses_websocket_error_event(
|
||||
http_error.status,
|
||||
kind.error_type,
|
||||
kind.code,
|
||||
&http_error.message,
|
||||
request_id,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn responses_websocket_error_event(
|
||||
status: StatusCode,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
message: &str,
|
||||
request_id: &str,
|
||||
) -> String {
|
||||
json!({
|
||||
"type": "error",
|
||||
"status": status.as_u16(),
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"code": code,
|
||||
"message": message,
|
||||
},
|
||||
"headers": {
|
||||
"x-request-id": request_id,
|
||||
},
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn upstream_client_websocket_error_event(
|
||||
error: &ResponseDispatchError,
|
||||
request_id: &str,
|
||||
) -> Option<String> {
|
||||
let ResponseDispatchError::Upstream(CodexClientError::Upstream { status, body, .. }) = error
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
if !status.is_client_error() {
|
||||
return None;
|
||||
}
|
||||
let Value::Object(mut payload) = serde_json::from_str::<Value>(body).ok()? else {
|
||||
return None;
|
||||
};
|
||||
if !payload.get("error").is_some_and(Value::is_object) {
|
||||
return None;
|
||||
}
|
||||
payload.insert("type".to_string(), Value::String("error".to_string()));
|
||||
payload.insert("status".to_string(), Value::Number(status.as_u16().into()));
|
||||
let headers = payload
|
||||
.entry("headers".to_string())
|
||||
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
||||
if !headers.is_object() {
|
||||
*headers = Value::Object(serde_json::Map::new());
|
||||
}
|
||||
headers.as_object_mut()?.insert(
|
||||
"x-request-id".to_string(),
|
||||
Value::String(request_id.to_string()),
|
||||
);
|
||||
Some(Value::Object(payload).to_string())
|
||||
}
|
||||
|
||||
fn responses_no_available_accounts_response() -> (StatusCode, Json<Value>) {
|
||||
let kind = NO_AVAILABLE_ACCOUNTS_ERROR;
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"type": "server_error",
|
||||
"code": "no_available_accounts",
|
||||
"message": NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE,
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
fn responses_error_response(status: StatusCode, message: &str) -> (StatusCode, Json<Value>) {
|
||||
let kind = responses_error_kind_for_status(status);
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"type": kind.error_type,
|
||||
"code": kind.code,
|
||||
"message": message,
|
||||
"message": NO_AVAILABLE_RESPONSES_ACCOUNTS_MESSAGE,
|
||||
}
|
||||
})),
|
||||
)
|
||||
@@ -208,11 +270,8 @@ fn invalid_openai_request_response(message: &str) -> (StatusCode, Json<Value>) {
|
||||
}
|
||||
|
||||
fn response_dispatch_openai_error(error: &ResponseDispatchError) -> OpenAiErrorDetails {
|
||||
let DispatchHttpError { status, message } = response_dispatch_http_error(
|
||||
error,
|
||||
ResponseDispatchStatusMode::UpstreamFailureStatus,
|
||||
ResponseDispatchMessageStyle::Standard,
|
||||
);
|
||||
let DispatchHttpError { status, message } =
|
||||
response_dispatch_http_error(error, ResponseDispatchMessageStyle::Standard);
|
||||
let kind = response_dispatch_error_kind(error);
|
||||
|
||||
OpenAiErrorDetails {
|
||||
@@ -226,17 +285,16 @@ fn response_dispatch_openai_error(error: &ResponseDispatchError) -> OpenAiErrorD
|
||||
fn response_dispatch_error_kind(error: &ResponseDispatchError) -> OpenAiErrorKind {
|
||||
dispatch_failure_openai_error_kind(
|
||||
error.metadata().failure_class,
|
||||
status_from_u16(error.http_status_code(), StatusCode::BAD_GATEWAY),
|
||||
status_from_u16(error.client_http_status_code(), StatusCode::BAD_GATEWAY),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn response_dispatch_http_error(
|
||||
fn response_dispatch_http_error(
|
||||
error: &ResponseDispatchError,
|
||||
status_mode: ResponseDispatchStatusMode,
|
||||
message_style: ResponseDispatchMessageStyle,
|
||||
) -> DispatchHttpError {
|
||||
DispatchHttpError {
|
||||
status: response_dispatch_status(error, status_mode),
|
||||
status: status_from_u16(error.client_http_status_code(), StatusCode::BAD_GATEWAY),
|
||||
message: response_dispatch_message(error, message_style),
|
||||
}
|
||||
}
|
||||
@@ -249,27 +307,12 @@ fn responses_error_kind_for_status(status: StatusCode) -> OpenAiErrorKind {
|
||||
} else if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
|
||||
AUTHENTICATION_ERROR
|
||||
} else if status.is_client_error() {
|
||||
OpenAiErrorKind {
|
||||
error_type: "invalid_request_error",
|
||||
code: "codex_api_error",
|
||||
}
|
||||
CODEX_CLIENT_ERROR
|
||||
} else {
|
||||
CODEX_API_ERROR
|
||||
}
|
||||
}
|
||||
|
||||
fn response_dispatch_status(
|
||||
error: &ResponseDispatchError,
|
||||
mode: ResponseDispatchStatusMode,
|
||||
) -> StatusCode {
|
||||
match (mode, error) {
|
||||
(ResponseDispatchStatusMode::UpstreamFailureStatus, ResponseDispatchError::Upstream(_)) => {
|
||||
StatusCode::BAD_GATEWAY
|
||||
}
|
||||
_ => status_from_u16(error.http_status_code(), StatusCode::BAD_GATEWAY),
|
||||
}
|
||||
}
|
||||
|
||||
fn response_dispatch_message(
|
||||
error: &ResponseDispatchError,
|
||||
style: ResponseDispatchMessageStyle,
|
||||
@@ -320,6 +363,8 @@ fn dispatch_failure_message(
|
||||
INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE.to_owned()
|
||||
}
|
||||
DispatchFailureClass::ResponseFailed => UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE.to_owned(),
|
||||
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_MESSAGE.to_owned(),
|
||||
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_MESSAGE.to_owned(),
|
||||
_ => UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE.to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -329,20 +374,28 @@ fn dispatch_failure_openai_error_kind(
|
||||
status: StatusCode,
|
||||
) -> OpenAiErrorKind {
|
||||
match failure_class {
|
||||
DispatchFailureClass::NoAvailableAccounts => UPSTREAM_UNAVAILABLE_ERROR,
|
||||
DispatchFailureClass::NoAvailableAccounts => NO_AVAILABLE_ACCOUNTS_ERROR,
|
||||
DispatchFailureClass::QuotaExhausted => INSUFFICIENT_QUOTA_ERROR,
|
||||
DispatchFailureClass::RateLimited => RATE_LIMIT_ERROR,
|
||||
DispatchFailureClass::Expired
|
||||
| DispatchFailureClass::Disabled
|
||||
| DispatchFailureClass::Banned => AUTHENTICATION_ERROR,
|
||||
DispatchFailureClass::ModelUnsupported => MODEL_NOT_FOUND_ERROR,
|
||||
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_ERROR,
|
||||
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_ERROR,
|
||||
DispatchFailureClass::InvalidSse
|
||||
| DispatchFailureClass::MissingCompleted
|
||||
| DispatchFailureClass::EmptyUpstreamResponse => INVALID_UPSTREAM_RESPONSE_ERROR,
|
||||
DispatchFailureClass::ResponseFailed => responses_error_kind_for_status(status),
|
||||
DispatchFailureClass::Upstream
|
||||
| DispatchFailureClass::CloudflareChallenge
|
||||
| DispatchFailureClass::CloudflarePathBlocked => UPSTREAM_ERROR,
|
||||
DispatchFailureClass::Upstream if status.is_client_error() => {
|
||||
responses_error_kind_for_status(status)
|
||||
}
|
||||
DispatchFailureClass::Upstream | DispatchFailureClass::UpstreamUnavailable => {
|
||||
UPSTREAM_ERROR
|
||||
}
|
||||
DispatchFailureClass::CloudflareChallenge | DispatchFailureClass::CloudflarePathBlocked => {
|
||||
UPSTREAM_ERROR
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! OpenAI 兼容 API 模块。
|
||||
|
||||
pub mod auth;
|
||||
pub mod errors;
|
||||
pub mod models;
|
||||
pub mod responses;
|
||||
pub mod routes;
|
||||
pub mod sse;
|
||||
pub mod router;
|
||||
@@ -1,16 +1,15 @@
|
||||
//! OpenAI 模型列表以及模型处理器。
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::IntoResponse,
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{
|
||||
proxy::auth::authorize_client_api_key, runtime::state::AppState,
|
||||
upstream::models::catalog::ModelCatalog,
|
||||
api::AppState, api::client::auth::authorize_client_api_key, models::catalog::ModelCatalog,
|
||||
};
|
||||
|
||||
use super::errors::{missing_client_api_key_response, model_not_found_response};
|
||||
@@ -1,36 +1,35 @@
|
||||
//! OpenAI Responses API 透明代理入口:解析原始 JSON body,提取调度元数据,
|
||||
//! 做最小 patch 后交给调度层,请求语义原样透传上游。
|
||||
|
||||
mod sse;
|
||||
mod websocket;
|
||||
|
||||
pub use websocket::responses_websocket;
|
||||
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
body::{Body, Bytes},
|
||||
extract::State,
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Extension, Json,
|
||||
};
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
http::middleware::request_id::{ClientIp, RequestId},
|
||||
proxy::{
|
||||
auth::authorize_client_api_key,
|
||||
dispatch::responses::{errors::ResponseDispatchError, service::ResponseDispatchStream},
|
||||
},
|
||||
runtime::state::AppState,
|
||||
upstream::protocol::{
|
||||
responses::{CodexCompactRequest, CodexResponsesRequest},
|
||||
sse::{encode_sse_event, sse_body_has_done},
|
||||
},
|
||||
api::AppState,
|
||||
api::middleware::request_id::{ClientIp, RequestId},
|
||||
dispatch::{errors::ResponseDispatchError, service::ResponseDispatchStream},
|
||||
upstream::openai::protocol::{responses::CodexResponsesRequest, sse::sse_body_has_done},
|
||||
};
|
||||
|
||||
use self::sse::{done_sse_frame, event_stream_response as sse_event_stream_response};
|
||||
use super::{
|
||||
errors::{
|
||||
invalid_responses_request_response, missing_client_api_key_response,
|
||||
model_not_found_response, responses_compact_dispatch_error_response,
|
||||
responses_dispatch_error_response, responses_stream_dispatch_failed_sse_event,
|
||||
model_not_found_response, responses_dispatch_error_response,
|
||||
responses_dispatch_error_response_ref, responses_stream_dispatch_failed_sse_event,
|
||||
},
|
||||
models::model_catalog_for_state,
|
||||
sse::{done_sse_frame, event_stream_response as sse_event_stream_response, SseResponseOptions},
|
||||
};
|
||||
|
||||
const OPENAI_SUBAGENT_HEADER: &str = "x-openai-subagent";
|
||||
@@ -104,33 +103,42 @@ pub fn build_codex_request(
|
||||
None => {}
|
||||
}
|
||||
apply_context_header_fields(&mut request, headers);
|
||||
apply_identity_fields(&mut request, headers);
|
||||
request.client_user_agent = user_agent_from_headers(headers);
|
||||
request
|
||||
}
|
||||
|
||||
/// 从客户端原始 Responses JSON body 构造 Codex compact 请求。
|
||||
///
|
||||
/// compact 端点只返回一次性 JSON(代理按非流式读取),故剥离 `stream` 这一
|
||||
/// transport 控制字段,避免上游返回代理无法解析的 SSE 形态。业务语义字段
|
||||
/// (`reasoning`/`text`/`store`/`prompt_cache_key`/`previous_response_id`/
|
||||
/// `include`/`client_metadata`/未知字段)一律原样透传。
|
||||
pub fn build_compact_request(
|
||||
mut body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
) -> CodexCompactRequest {
|
||||
body.remove(COMPACT_STREAM_KEY);
|
||||
for key in TRANSPORT_ONLY_KEYS {
|
||||
body.remove(key);
|
||||
}
|
||||
CodexCompactRequest {
|
||||
body,
|
||||
client_ip: None,
|
||||
client_user_agent: user_agent_from_headers(headers),
|
||||
}
|
||||
fn apply_identity_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) {
|
||||
request.client_conversation_id = request.prompt_cache_key().map(ToString::to_string);
|
||||
request.client_session_id = identity_context_string(request.body(), "session_id")
|
||||
.or_else(|| header_string(headers, "session-id"));
|
||||
request.client_thread_id = identity_context_string(request.body(), "thread_id")
|
||||
.or_else(|| header_string(headers, "thread-id"));
|
||||
request.client_request_id = identity_context_string(request.body(), "x-client-request-id")
|
||||
.or_else(|| header_string(headers, "x-client-request-id"));
|
||||
request.client_turn_id = identity_context_string(request.body(), "turn_id")
|
||||
.or_else(|| header_string(headers, "x-codex-turn-id"));
|
||||
request.codex_window_id = request.codex_window_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-window-id")
|
||||
.or_else(|| header_string(headers, "x-codex-window-id"))
|
||||
});
|
||||
request.parent_thread_id = request.parent_thread_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-parent-thread-id")
|
||||
.or_else(|| header_string(headers, "x-codex-parent-thread-id"))
|
||||
});
|
||||
}
|
||||
|
||||
/// compact 端点不支持流式,转发前剥离 `stream`(transport 控制字段,非业务语义)。
|
||||
const COMPACT_STREAM_KEY: &str = "stream";
|
||||
fn identity_context_string(body: &Map<String, Value>, key: &str) -> Option<String> {
|
||||
body_context_string(body, key).or_else(|| {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(key))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
})
|
||||
}
|
||||
|
||||
fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
@@ -193,41 +201,6 @@ impl ContextField {
|
||||
}
|
||||
}
|
||||
|
||||
/// 编码 OpenAI Responses `response.failed` SSE 事件。
|
||||
pub fn response_failed_sse_event(error_type: &str, code: &str, message: &str) -> String {
|
||||
response_failed_sse_event_with_id(None, error_type, code, message)
|
||||
}
|
||||
|
||||
/// 使用指定 response id 编码 OpenAI Responses `response.failed` SSE 事件。
|
||||
pub fn response_failed_sse_event_with_id(
|
||||
response_id: Option<&str>,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
message: &str,
|
||||
) -> String {
|
||||
let error = json!({
|
||||
"type": error_type,
|
||||
"code": code,
|
||||
"message": message,
|
||||
});
|
||||
let response_id = response_id
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map_or_else(
|
||||
|| format!("resp_proxy_{}", uuid::Uuid::new_v4().simple()),
|
||||
ToString::to_string,
|
||||
);
|
||||
let data = json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"status": "failed",
|
||||
"error": error,
|
||||
},
|
||||
"error": error,
|
||||
});
|
||||
encode_sse_event("response.failed", &data.to_string())
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// HTTP 处理器
|
||||
// ====================================================================
|
||||
@@ -243,7 +216,7 @@ pub async fn responses(
|
||||
handle_responses(
|
||||
state,
|
||||
request_id,
|
||||
client_ip_string(client_ip),
|
||||
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string()),
|
||||
headers,
|
||||
body,
|
||||
"/v1/responses",
|
||||
@@ -263,7 +236,7 @@ pub async fn review_responses(
|
||||
handle_responses(
|
||||
state,
|
||||
request_id,
|
||||
client_ip_string(client_ip),
|
||||
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string()),
|
||||
headers,
|
||||
body,
|
||||
"/v1/responses/review",
|
||||
@@ -272,29 +245,6 @@ pub async fn review_responses(
|
||||
.await
|
||||
}
|
||||
|
||||
/// `POST /v1/responses/compact`
|
||||
pub async fn compact_responses(
|
||||
State(state): State<AppState>,
|
||||
Extension(request_id): Extension<RequestId>,
|
||||
client_ip: Option<Extension<ClientIp>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_compact_responses(
|
||||
state,
|
||||
request_id,
|
||||
client_ip_string(client_ip),
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 从可选的 `ClientIp` extension 提取 IP 字符串。
|
||||
fn client_ip_string(client_ip: Option<Extension<ClientIp>>) -> Option<String> {
|
||||
client_ip.map(|Extension(client_ip)| client_ip.as_str().to_string())
|
||||
}
|
||||
|
||||
async fn handle_responses(
|
||||
state: AppState,
|
||||
request_id: RequestId,
|
||||
@@ -304,35 +254,48 @@ async fn handle_responses(
|
||||
route: &str,
|
||||
forced_subagent: Option<&str>,
|
||||
) -> Response {
|
||||
if !authorize_client_api_key(&state, &headers).await {
|
||||
let Some(client_api_key_id) =
|
||||
crate::api::client::auth::authorized_client_api_key_id(&state, &headers).await
|
||||
else {
|
||||
return missing_client_api_key_response().into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let Some(body) = parse_responses_body(&body) else {
|
||||
return invalid_responses_request_response().into_response();
|
||||
};
|
||||
let model = match validated_responses_model(&state, request_model(&body)).await {
|
||||
Ok(model) => model,
|
||||
Err(ResponsesModelValidationError::InvalidRequest) => {
|
||||
let prepared = match prepare_responses_request(
|
||||
&state,
|
||||
body,
|
||||
&headers,
|
||||
forced_subagent,
|
||||
client_ip,
|
||||
client_api_key_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(ResponsesRequestValidationError::InvalidRequest) => {
|
||||
return invalid_responses_request_response().into_response();
|
||||
}
|
||||
Err(ResponsesModelValidationError::ModelNotFound) => {
|
||||
Err(ResponsesRequestValidationError::ModelNotFound) => {
|
||||
return model_not_found_response().into_response();
|
||||
}
|
||||
};
|
||||
let mut codex_request = build_codex_request(body, &headers, forced_subagent);
|
||||
codex_request.client_ip = client_ip;
|
||||
let PreparedResponsesRequest {
|
||||
mut request,
|
||||
requested_model,
|
||||
} = prepared;
|
||||
// 客户端未显式指定 transport 时默认偏好 WebSocket。
|
||||
if !codex_request.force_http_sse {
|
||||
codex_request.use_websocket = true;
|
||||
if !request.force_http_sse {
|
||||
request.use_websocket = true;
|
||||
}
|
||||
let stream = codex_request.stream();
|
||||
let stream = request.stream();
|
||||
|
||||
if stream {
|
||||
return match state
|
||||
.services
|
||||
.responses
|
||||
.stream(request_id.as_str(), route, codex_request, &model)
|
||||
.stream(request_id.as_str(), route, request, &requested_model)
|
||||
.await
|
||||
{
|
||||
Ok(stream) => live_event_stream_response(stream),
|
||||
@@ -343,51 +306,17 @@ async fn handle_responses(
|
||||
match state
|
||||
.services
|
||||
.responses
|
||||
.complete(request_id.as_str(), route, codex_request, &model)
|
||||
.complete(request_id.as_str(), route, request, &requested_model)
|
||||
.await
|
||||
{
|
||||
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
|
||||
Ok(result) => apply_safe_response_headers(
|
||||
(StatusCode::OK, Json(result.body)).into_response(),
|
||||
result.response_headers,
|
||||
),
|
||||
Err(error) => responses_dispatch_error_response(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_compact_responses(
|
||||
state: AppState,
|
||||
request_id: RequestId,
|
||||
client_ip: Option<String>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
if !authorize_client_api_key(&state, &headers).await {
|
||||
return missing_client_api_key_response().into_response();
|
||||
}
|
||||
|
||||
let Some(body) = parse_responses_body(&body) else {
|
||||
return invalid_responses_request_response().into_response();
|
||||
};
|
||||
let model = match validated_responses_model(&state, request_model(&body)).await {
|
||||
Ok(model) => model,
|
||||
Err(ResponsesModelValidationError::InvalidRequest) => {
|
||||
return invalid_responses_request_response().into_response();
|
||||
}
|
||||
Err(ResponsesModelValidationError::ModelNotFound) => {
|
||||
return model_not_found_response().into_response();
|
||||
}
|
||||
};
|
||||
let mut compact_request = build_compact_request(body, &headers);
|
||||
compact_request.client_ip = client_ip;
|
||||
|
||||
match state
|
||||
.services
|
||||
.responses
|
||||
.compact(request_id.as_str(), compact_request, &model)
|
||||
.await
|
||||
{
|
||||
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
|
||||
Err(error) => responses_compact_dispatch_error_response(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析请求体:必须是 JSON object,否则视为 invalid request。
|
||||
fn parse_responses_body(body: &Bytes) -> Option<Map<String, Value>> {
|
||||
match serde_json::from_slice::<Value>(body) {
|
||||
@@ -402,23 +331,47 @@ fn request_model(body: &Map<String, Value>) -> &str {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(super) struct PreparedResponsesRequest {
|
||||
pub(super) request: CodexResponsesRequest,
|
||||
pub(super) requested_model: String,
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_responses_request(
|
||||
state: &AppState,
|
||||
body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
forced_subagent: Option<&str>,
|
||||
client_ip: Option<String>,
|
||||
client_api_key_id: String,
|
||||
) -> Result<PreparedResponsesRequest, ResponsesRequestValidationError> {
|
||||
let requested_model = validated_responses_model(state, request_model(&body)).await?;
|
||||
let mut request = build_codex_request(body, headers, forced_subagent);
|
||||
request.client_ip = client_ip;
|
||||
request.client_api_key_id = Some(client_api_key_id);
|
||||
Ok(PreparedResponsesRequest {
|
||||
request,
|
||||
requested_model,
|
||||
})
|
||||
}
|
||||
|
||||
async fn validated_responses_model(
|
||||
state: &AppState,
|
||||
raw_model: &str,
|
||||
) -> Result<String, ResponsesModelValidationError> {
|
||||
) -> Result<String, ResponsesRequestValidationError> {
|
||||
let model = raw_model.trim();
|
||||
if model.is_empty() {
|
||||
return Err(ResponsesModelValidationError::InvalidRequest);
|
||||
return Err(ResponsesRequestValidationError::InvalidRequest);
|
||||
}
|
||||
let catalog = model_catalog_for_state(state).await;
|
||||
if catalog.is_recognized_model_name(model) {
|
||||
if catalog.is_empty() || catalog.is_recognized_model_name(model) {
|
||||
Ok(model.to_string())
|
||||
} else {
|
||||
Err(ResponsesModelValidationError::ModelNotFound)
|
||||
Err(ResponsesRequestValidationError::ModelNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
enum ResponsesModelValidationError {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ResponsesRequestValidationError {
|
||||
InvalidRequest,
|
||||
ModelNotFound,
|
||||
}
|
||||
@@ -428,15 +381,11 @@ pub(crate) fn user_agent_from_headers(headers: &HeaderMap) -> Option<String> {
|
||||
}
|
||||
|
||||
fn openai_subagent_from_headers(headers: &HeaderMap) -> Option<String> {
|
||||
header_string(headers, OPENAI_SUBAGENT_HEADER).and_then(|value| {
|
||||
if matches!(
|
||||
header_string(headers, OPENAI_SUBAGENT_HEADER).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"review" | "compact" | "memory_consolidation" | "collab_spawn"
|
||||
) {
|
||||
Some(value)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -451,14 +400,40 @@ fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
|
||||
fn event_stream_response(mut body: String) -> Response {
|
||||
ensure_done_sse_frame(&mut body);
|
||||
sse_event_stream_response(Body::from(body), SseResponseOptions::BASIC)
|
||||
sse_event_stream_response(Body::from(body))
|
||||
}
|
||||
|
||||
fn live_event_stream_response(stream: ResponseDispatchStream) -> Response {
|
||||
sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC)
|
||||
let response = sse_event_stream_response(Body::from_stream(stream.body));
|
||||
apply_safe_response_headers(response, stream.response_headers)
|
||||
}
|
||||
|
||||
fn apply_safe_response_headers(mut response: Response, headers: Vec<(String, String)>) -> Response {
|
||||
for (name, value) in headers {
|
||||
let name = match name.as_str() {
|
||||
"x-request-id" => axum::http::header::HeaderName::from_static("x-request-id"),
|
||||
"openai-model" => axum::http::header::HeaderName::from_static("openai-model"),
|
||||
"x-models-etag" => axum::http::header::HeaderName::from_static("x-models-etag"),
|
||||
"x-reasoning-included" => {
|
||||
axum::http::header::HeaderName::from_static("x-reasoning-included")
|
||||
}
|
||||
"openai-processing-ms" => {
|
||||
axum::http::header::HeaderName::from_static("openai-processing-ms")
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let Ok(value) = axum::http::HeaderValue::from_str(&value) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn response_dispatch_stream_error_response(error: &ResponseDispatchError) -> Response {
|
||||
if (400..=499).contains(&error.client_http_status_code()) {
|
||||
return responses_dispatch_error_response_ref(error);
|
||||
}
|
||||
event_stream_response(responses_stream_dispatch_failed_sse_event(error))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
//! OpenAI SSE 封装。
|
||||
//!
|
||||
//! 提供 SSE 事件编码与流式响应构建工具。
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{
|
||||
StatusCode,
|
||||
header::{CACHE_CONTROL, CONTENT_TYPE},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
|
||||
use crate::upstream::openai::protocol::sse::DONE_SSE_FRAME;
|
||||
|
||||
use crate::api::client::errors::openai_error_response;
|
||||
|
||||
/// 编码 OpenAI 流结束标记。
|
||||
pub fn done_sse_frame() -> &'static str {
|
||||
DONE_SSE_FRAME
|
||||
}
|
||||
|
||||
pub fn event_stream_response(body: Body) -> Response {
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(CONTENT_TYPE, "text/event-stream")
|
||||
.header(CACHE_CONTROL, "no-cache")
|
||||
.body(body)
|
||||
.unwrap_or_else(|_| {
|
||||
openai_error_response(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"Failed to build stream response",
|
||||
"server_error",
|
||||
"stream_response_error",
|
||||
)
|
||||
.into_response()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,688 @@
|
||||
//! 官方 Responses WebSocket 入站协议。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::{Request, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use hyper::upgrade::Upgraded;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
use tokio_tungstenite::{
|
||||
WebSocketStream,
|
||||
tungstenite::{
|
||||
Error as WebSocketError, Message,
|
||||
handshake::server::create_response_with_body,
|
||||
protocol::{CloseFrame, Role, WebSocketConfig, frame::coding::CloseCode},
|
||||
},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
api::{AppState, middleware::request_id::RequestId},
|
||||
dispatch::{recovery::history::sanitize_replay_items, service::ResponseDispatchStream},
|
||||
upstream::openai::protocol::{
|
||||
sse::{SseEvent, SseEventDecoder},
|
||||
websocket::{is_terminal_websocket_event, websocket_event_type},
|
||||
},
|
||||
};
|
||||
|
||||
use super::{PreparedResponsesRequest, ResponsesRequestValidationError, prepare_responses_request};
|
||||
use crate::api::client::{
|
||||
auth::authorized_client_api_key_id,
|
||||
errors::{
|
||||
missing_client_api_key_response, openai_error_response,
|
||||
responses_websocket_dispatch_error_event, responses_websocket_error_event,
|
||||
},
|
||||
router::MAX_CLIENT_REQUEST_BODY_BYTES,
|
||||
};
|
||||
|
||||
const RESPONSES_ROUTE: &str = "/v1/responses";
|
||||
const INVALID_REQUEST_MESSAGE: &str = "Invalid responses request";
|
||||
const MODEL_NOT_FOUND_MESSAGE: &str = "Model not found";
|
||||
const INVALID_STREAM_MESSAGE: &str = "Responses WebSocket requests require stream=true";
|
||||
const INVALID_UPSTREAM_EVENT_MESSAGE: &str = "Invalid upstream Responses event";
|
||||
const STREAM_DISCONNECTED_MESSAGE: &str = "Upstream stream closed before response.completed";
|
||||
|
||||
type ResponsesWebSocket = WebSocketStream<TokioIo<Upgraded>>;
|
||||
|
||||
/// 将已认证的 `GET /v1/responses` 请求升级为官方 Responses WebSocket 协议。
|
||||
pub async fn responses_websocket(State(state): State<AppState>, mut request: Request) -> Response {
|
||||
let headers = request.headers().clone();
|
||||
let Some(client_api_key_id) = authorized_client_api_key_id(&state, &headers).await else {
|
||||
return missing_client_api_key_response().into_response();
|
||||
};
|
||||
let connection_request_id = request
|
||||
.extensions()
|
||||
.get::<RequestId>()
|
||||
.cloned()
|
||||
.unwrap_or_else(RequestId::generate);
|
||||
let client_ip = request
|
||||
.extensions()
|
||||
.get::<crate::api::middleware::request_id::ClientIp>()
|
||||
.map(|client_ip| client_ip.as_str().to_string());
|
||||
let connection_id = connection_request_id.as_str().to_string();
|
||||
let response = match create_response_with_body(&request, Body::empty) {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"invalid Responses WebSocket upgrade request"
|
||||
);
|
||||
return openai_error_response(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid Responses WebSocket upgrade request",
|
||||
"invalid_request_error",
|
||||
"invalid_websocket_upgrade",
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
let upgrade = hyper::upgrade::on(&mut request);
|
||||
let connection_drain = state.services.connection_drain.clone();
|
||||
connection_drain.spawn(async move {
|
||||
match upgrade.await {
|
||||
Ok(upgraded) => {
|
||||
let socket = WebSocketStream::from_raw_socket(
|
||||
TokioIo::new(upgraded),
|
||||
Role::Server,
|
||||
Some(downstream_websocket_config()),
|
||||
)
|
||||
.await;
|
||||
serve_responses_websocket(
|
||||
socket,
|
||||
ResponsesWebSocketSession {
|
||||
state,
|
||||
headers,
|
||||
client_ip,
|
||||
client_api_key_id,
|
||||
connection_id,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket upgrade failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
response
|
||||
}
|
||||
|
||||
fn downstream_websocket_config() -> WebSocketConfig {
|
||||
let mut config = WebSocketConfig::default();
|
||||
config.max_message_size = Some(MAX_CLIENT_REQUEST_BODY_BYTES);
|
||||
config.max_frame_size = Some(MAX_CLIENT_REQUEST_BODY_BYTES);
|
||||
config
|
||||
}
|
||||
|
||||
struct ResponsesWebSocketSession {
|
||||
state: AppState,
|
||||
headers: HeaderMap,
|
||||
client_ip: Option<String>,
|
||||
client_api_key_id: String,
|
||||
connection_id: String,
|
||||
}
|
||||
|
||||
struct ConnectionReplayState {
|
||||
last_response_id: Option<String>,
|
||||
input: Option<Arc<Vec<Value>>>,
|
||||
}
|
||||
|
||||
enum PendingReplayUpdate {
|
||||
Replace(Vec<Value>),
|
||||
Append(Vec<Value>),
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
struct CompletedTurn {
|
||||
response_id: String,
|
||||
output: Vec<Value>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CompletedTurnCollector {
|
||||
completed_items: Vec<Value>,
|
||||
completed: Option<CompletedTurn>,
|
||||
}
|
||||
|
||||
enum ForwardDispatchOutcome {
|
||||
Continue(Option<CompletedTurn>),
|
||||
Disconnect,
|
||||
}
|
||||
|
||||
impl ConnectionReplayState {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
last_response_id: None,
|
||||
input: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare(
|
||||
&self,
|
||||
request: &mut crate::upstream::openai::protocol::responses::CodexResponsesRequest,
|
||||
) -> PendingReplayUpdate {
|
||||
let turn_input = request.input().to_vec();
|
||||
match request.previous_response_id() {
|
||||
None => {
|
||||
request.local_replay_input = None;
|
||||
PendingReplayUpdate::Replace(turn_input)
|
||||
}
|
||||
Some(previous_response_id)
|
||||
if self.last_response_id.as_deref() == Some(previous_response_id)
|
||||
&& self.input.is_some() =>
|
||||
{
|
||||
request.local_replay_input.clone_from(&self.input);
|
||||
PendingReplayUpdate::Append(turn_input)
|
||||
}
|
||||
Some(_) => {
|
||||
request.local_replay_input = None;
|
||||
PendingReplayUpdate::Unavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn commit(&mut self, pending: PendingReplayUpdate, completed: CompletedTurn) {
|
||||
self.last_response_id = Some(completed.response_id);
|
||||
match pending {
|
||||
PendingReplayUpdate::Replace(input) => self.replace(input, completed.output),
|
||||
PendingReplayUpdate::Append(input) => self.append(input, completed.output),
|
||||
PendingReplayUpdate::Unavailable => self.clear_input(),
|
||||
}
|
||||
}
|
||||
|
||||
fn replace(&mut self, mut input: Vec<Value>, mut output: Vec<Value>) {
|
||||
sanitize_replay_items(&mut input);
|
||||
sanitize_replay_items(&mut output);
|
||||
input.append(&mut output);
|
||||
self.input = Some(Arc::new(input));
|
||||
}
|
||||
|
||||
fn append(&mut self, mut input: Vec<Value>, mut output: Vec<Value>) {
|
||||
sanitize_replay_items(&mut input);
|
||||
sanitize_replay_items(&mut output);
|
||||
input.append(&mut output);
|
||||
let Some(history) = self.input.as_mut() else {
|
||||
return;
|
||||
};
|
||||
Arc::make_mut(history).append(&mut input);
|
||||
}
|
||||
|
||||
fn clear_input(&mut self) {
|
||||
self.input = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl CompletedTurnCollector {
|
||||
fn observe(&mut self, event: &SseEvent) {
|
||||
let Ok(value) = serde_json::from_str::<Value>(&event.data) else {
|
||||
return;
|
||||
};
|
||||
let event_type = event
|
||||
.event
|
||||
.as_deref()
|
||||
.or_else(|| value.get("type").and_then(Value::as_str));
|
||||
match event_type {
|
||||
Some("response.output_item.done") => {
|
||||
if let Some(item) = value.get("item") {
|
||||
self.completed_items.push(item.clone());
|
||||
}
|
||||
}
|
||||
Some("response.completed") => {
|
||||
let Some(response_id) = value
|
||||
.pointer("/response/id")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let output = value
|
||||
.pointer("/response/output")
|
||||
.and_then(Value::as_array)
|
||||
.filter(|output| !output.is_empty())
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.completed_items.clone());
|
||||
self.completed = Some(CompletedTurn {
|
||||
response_id,
|
||||
output,
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve_responses_websocket(
|
||||
mut socket: ResponsesWebSocket,
|
||||
session: ResponsesWebSocketSession,
|
||||
) {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %session.connection_id,
|
||||
"Responses WebSocket connected"
|
||||
);
|
||||
let mut request_count = 0u64;
|
||||
let mut replay_state = ConnectionReplayState::new();
|
||||
|
||||
while let Some(message) = socket.next().await {
|
||||
let payload = match message {
|
||||
Ok(Message::Text(payload)) => payload,
|
||||
Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) => continue,
|
||||
Ok(Message::Close(_)) => break,
|
||||
Ok(Message::Binary(_)) => {
|
||||
close_for_protocol_violation(
|
||||
&mut socket,
|
||||
"Responses WebSocket accepts text frames only",
|
||||
)
|
||||
.await;
|
||||
break;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %session.connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket receive failed"
|
||||
);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
request_count = request_count.saturating_add(1);
|
||||
let request_id = RequestId::generate();
|
||||
let body = match parse_response_create(payload.as_str()) {
|
||||
Ok(body) => body,
|
||||
Err(error) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %session.connection_id,
|
||||
request_id = %request_id.as_str(),
|
||||
error = %error,
|
||||
"Responses WebSocket request rejected"
|
||||
);
|
||||
if !send_error(
|
||||
&mut socket,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request_error",
|
||||
"invalid_request",
|
||||
&error.to_string(),
|
||||
request_id.as_str(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut prepared = match prepare_responses_request(
|
||||
&session.state,
|
||||
body,
|
||||
&session.headers,
|
||||
None,
|
||||
session.client_ip.clone(),
|
||||
session.client_api_key_id.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => {
|
||||
let (status, code, message) = match error {
|
||||
ResponsesRequestValidationError::InvalidRequest => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request",
|
||||
INVALID_REQUEST_MESSAGE,
|
||||
),
|
||||
ResponsesRequestValidationError::ModelNotFound => (
|
||||
StatusCode::NOT_FOUND,
|
||||
"model_not_found",
|
||||
MODEL_NOT_FOUND_MESSAGE,
|
||||
),
|
||||
};
|
||||
if !send_error(
|
||||
&mut socket,
|
||||
status,
|
||||
"invalid_request_error",
|
||||
code,
|
||||
message,
|
||||
request_id.as_str(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let pending_replay = replay_state.prepare(&mut prepared.request);
|
||||
|
||||
match dispatch_response_create(&mut socket, &session, request_id, prepared).await {
|
||||
ForwardDispatchOutcome::Continue(Some(completed)) => {
|
||||
replay_state.commit(pending_replay, completed);
|
||||
}
|
||||
ForwardDispatchOutcome::Continue(None) => {}
|
||||
ForwardDispatchOutcome::Disconnect => break,
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
websocket_connection_id = %session.connection_id,
|
||||
request_count,
|
||||
"Responses WebSocket disconnected"
|
||||
);
|
||||
}
|
||||
|
||||
async fn dispatch_response_create(
|
||||
socket: &mut ResponsesWebSocket,
|
||||
session: &ResponsesWebSocketSession,
|
||||
request_id: RequestId,
|
||||
prepared: PreparedResponsesRequest,
|
||||
) -> ForwardDispatchOutcome {
|
||||
let PreparedResponsesRequest {
|
||||
mut request,
|
||||
requested_model,
|
||||
} = prepared;
|
||||
if !request.force_http_sse {
|
||||
request.use_websocket = true;
|
||||
}
|
||||
let stream = session
|
||||
.state
|
||||
.services
|
||||
.responses
|
||||
.stream(
|
||||
request_id.as_str(),
|
||||
RESPONSES_ROUTE,
|
||||
request,
|
||||
&requested_model,
|
||||
)
|
||||
.await;
|
||||
let stream = match stream {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
let event = responses_websocket_dispatch_error_event(&error, request_id.as_str());
|
||||
return if send_text(socket, event).await {
|
||||
ForwardDispatchOutcome::Continue(None)
|
||||
} else {
|
||||
ForwardDispatchOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
};
|
||||
forward_dispatch_stream(socket, stream, request_id.as_str()).await
|
||||
}
|
||||
|
||||
async fn forward_dispatch_stream(
|
||||
socket: &mut ResponsesWebSocket,
|
||||
stream: ResponseDispatchStream,
|
||||
request_id: &str,
|
||||
) -> ForwardDispatchOutcome {
|
||||
let ResponseDispatchStream {
|
||||
body,
|
||||
response_headers,
|
||||
} = stream;
|
||||
if !send_text(
|
||||
socket,
|
||||
response_metadata_event(request_id, &response_headers),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
|
||||
let mut body = body;
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
let mut terminal_sent = false;
|
||||
let mut completed = CompletedTurnCollector::default();
|
||||
loop {
|
||||
let next = if terminal_sent {
|
||||
ActiveResponseInput::Upstream(body.next().await)
|
||||
} else {
|
||||
tokio::select! {
|
||||
next = body.next() => ActiveResponseInput::Upstream(next),
|
||||
message = socket.next() => ActiveResponseInput::Downstream(message),
|
||||
}
|
||||
};
|
||||
match next {
|
||||
ActiveResponseInput::Upstream(Some(Ok(chunk))) => {
|
||||
let events = match decoder.push(&chunk) {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
tracing::warn!(request_id, error = %error, "failed to decode Responses SSE for WebSocket forwarding");
|
||||
let sent = send_error(
|
||||
socket,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"invalid_upstream_response",
|
||||
INVALID_UPSTREAM_EVENT_MESSAGE,
|
||||
request_id,
|
||||
)
|
||||
.await;
|
||||
drop(body);
|
||||
return if sent {
|
||||
ForwardDispatchOutcome::Continue(None)
|
||||
} else {
|
||||
ForwardDispatchOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
};
|
||||
if !forward_sse_events(
|
||||
socket,
|
||||
events,
|
||||
request_id,
|
||||
&mut terminal_sent,
|
||||
&mut completed,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
}
|
||||
ActiveResponseInput::Upstream(Some(Err(error))) => {
|
||||
tracing::warn!(request_id, error = %error, "Responses dispatch stream failed");
|
||||
if terminal_sent {
|
||||
return ForwardDispatchOutcome::Continue(completed.completed);
|
||||
}
|
||||
return if send_error(
|
||||
socket,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"upstream_error",
|
||||
STREAM_DISCONNECTED_MESSAGE,
|
||||
request_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ForwardDispatchOutcome::Continue(None)
|
||||
} else {
|
||||
ForwardDispatchOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
ActiveResponseInput::Upstream(None) => {
|
||||
let events = match decoder.finish() {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
tracing::warn!(request_id, error = %error, "failed to finish Responses SSE decoding");
|
||||
return if send_error(
|
||||
socket,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"invalid_upstream_response",
|
||||
INVALID_UPSTREAM_EVENT_MESSAGE,
|
||||
request_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ForwardDispatchOutcome::Continue(None)
|
||||
} else {
|
||||
ForwardDispatchOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
};
|
||||
if !forward_sse_events(
|
||||
socket,
|
||||
events,
|
||||
request_id,
|
||||
&mut terminal_sent,
|
||||
&mut completed,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
if terminal_sent {
|
||||
return ForwardDispatchOutcome::Continue(completed.completed);
|
||||
}
|
||||
return if send_error(
|
||||
socket,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"stream_disconnected",
|
||||
STREAM_DISCONNECTED_MESSAGE,
|
||||
request_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ForwardDispatchOutcome::Continue(None)
|
||||
} else {
|
||||
ForwardDispatchOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
ActiveResponseInput::Downstream(Some(Ok(
|
||||
Message::Ping(_) | Message::Pong(_) | Message::Frame(_),
|
||||
))) => {}
|
||||
ActiveResponseInput::Downstream(Some(Ok(Message::Close(_))) | None) => {
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
ActiveResponseInput::Downstream(Some(Ok(Message::Text(_) | Message::Binary(_)))) => {
|
||||
close_for_protocol_violation(
|
||||
socket,
|
||||
"Only one response.create may be active per connection",
|
||||
)
|
||||
.await;
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
ActiveResponseInput::Downstream(Some(Err(error))) => {
|
||||
tracing::info!(request_id, error = %error, "Responses WebSocket client disconnected during response");
|
||||
return ForwardDispatchOutcome::Disconnect;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum ActiveResponseInput {
|
||||
Upstream(Option<Result<bytes::Bytes, crate::dispatch::errors::ResponseDispatchStreamError>>),
|
||||
Downstream(Option<Result<Message, WebSocketError>>),
|
||||
}
|
||||
|
||||
async fn forward_sse_events(
|
||||
socket: &mut ResponsesWebSocket,
|
||||
events: Vec<SseEvent>,
|
||||
request_id: &str,
|
||||
terminal_sent: &mut bool,
|
||||
completed: &mut CompletedTurnCollector,
|
||||
) -> bool {
|
||||
for event in events {
|
||||
let Some(event_type) = websocket_event_type(&event.data) else {
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
"Responses SSE event does not contain a valid JSON type"
|
||||
);
|
||||
let _ = send_error(
|
||||
socket,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"server_error",
|
||||
"invalid_upstream_response",
|
||||
INVALID_UPSTREAM_EVENT_MESSAGE,
|
||||
request_id,
|
||||
)
|
||||
.await;
|
||||
return false;
|
||||
};
|
||||
completed.observe(&event);
|
||||
if !send_text(socket, event.data).await {
|
||||
return false;
|
||||
}
|
||||
*terminal_sent |= is_terminal_websocket_event(&event_type);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn response_metadata_event(request_id: &str, response_headers: &[(String, String)]) -> String {
|
||||
let mut headers = response_headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), Value::String(value.clone())))
|
||||
.collect::<Map<String, Value>>();
|
||||
headers.insert(
|
||||
"x-request-id".to_string(),
|
||||
Value::String(request_id.to_string()),
|
||||
);
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": headers,
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn parse_response_create(payload: &str) -> Result<Map<String, Value>, ResponseCreateFrameError> {
|
||||
let Value::Object(mut body) = serde_json::from_str::<Value>(payload)? else {
|
||||
return Err(ResponseCreateFrameError::ExpectedObject);
|
||||
};
|
||||
match body.remove("type") {
|
||||
Some(Value::String(message_type)) if message_type == "response.create" => {}
|
||||
_ => return Err(ResponseCreateFrameError::UnsupportedType),
|
||||
}
|
||||
if body.get("stream").and_then(Value::as_bool) != Some(true) {
|
||||
return Err(ResponseCreateFrameError::StreamingRequired);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
enum ResponseCreateFrameError {
|
||||
#[error("Invalid response.create JSON: {0}")]
|
||||
InvalidJson(#[from] serde_json::Error),
|
||||
#[error("A response.create payload must be a JSON object")]
|
||||
ExpectedObject,
|
||||
#[error("Unsupported Responses WebSocket message type")]
|
||||
UnsupportedType,
|
||||
#[error("{INVALID_STREAM_MESSAGE}")]
|
||||
StreamingRequired,
|
||||
}
|
||||
|
||||
async fn send_error(
|
||||
socket: &mut ResponsesWebSocket,
|
||||
status: StatusCode,
|
||||
error_type: &str,
|
||||
code: &str,
|
||||
message: &str,
|
||||
request_id: &str,
|
||||
) -> bool {
|
||||
send_text(
|
||||
socket,
|
||||
responses_websocket_error_event(status, error_type, code, message, request_id),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn send_text(socket: &mut ResponsesWebSocket, payload: String) -> bool {
|
||||
if let Err(error) = socket.send(Message::Text(payload.into())).await {
|
||||
tracing::info!(error = %error, "Responses WebSocket send failed");
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn close_for_protocol_violation(socket: &mut ResponsesWebSocket, reason: &'static str) {
|
||||
let _ = socket
|
||||
.send(Message::Close(Some(CloseFrame {
|
||||
code: CloseCode::Policy,
|
||||
reason: reason.into(),
|
||||
})))
|
||||
.await;
|
||||
}
|
||||
@@ -1,25 +1,28 @@
|
||||
//! OpenAI 兼容 API 路由。
|
||||
|
||||
use axum::{
|
||||
routing::{get, post},
|
||||
Router,
|
||||
extract::DefaultBodyLimit,
|
||||
routing::{get, post},
|
||||
};
|
||||
|
||||
use crate::runtime::state::AppState;
|
||||
use crate::api::AppState;
|
||||
|
||||
use super::{
|
||||
models::{model_catalog, model_detail, model_info, models},
|
||||
responses::{compact_responses, responses, review_responses},
|
||||
responses::{responses, responses_websocket, review_responses},
|
||||
};
|
||||
|
||||
pub const MAX_CLIENT_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// 构造 OpenAI 兼容 API 路由。
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/v1/responses", post(responses))
|
||||
.route("/v1/responses", get(responses_websocket).post(responses))
|
||||
.route("/v1/responses/review", post(review_responses))
|
||||
.route("/v1/responses/compact", post(compact_responses))
|
||||
.route("/v1/models", get(models))
|
||||
.route("/v1/models/catalog", get(model_catalog))
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//! 进程关闭时统一排空入站 HTTP 流与 WebSocket 任务。
|
||||
|
||||
use std::future::Future;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::{Request, State},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use tokio_util::{sync::CancellationToken, task::TaskTracker};
|
||||
|
||||
/// 入站连接排空控制器。
|
||||
#[derive(Clone)]
|
||||
pub struct ConnectionDrain {
|
||||
cancellation: CancellationToken,
|
||||
connection_tasks: TaskTracker,
|
||||
}
|
||||
|
||||
impl Default for ConnectionDrain {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
cancellation: CancellationToken::new(),
|
||||
connection_tasks: TaskTracker::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ConnectionDrain {
|
||||
/// 注册需要随进程关闭而终止的长连接任务。
|
||||
pub fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
|
||||
let cancellation = self.cancellation.clone();
|
||||
drop(self.connection_tasks.spawn(async move {
|
||||
cancellation.run_until_cancelled(future).await;
|
||||
}));
|
||||
}
|
||||
|
||||
/// 停止所有受管连接,并返回关闭前仍在运行的长连接任务数。
|
||||
pub fn begin_shutdown(&self) -> usize {
|
||||
let active_websocket_connections = self.connection_tasks.len();
|
||||
self.connection_tasks.close();
|
||||
self.cancellation.cancel();
|
||||
active_websocket_connections
|
||||
}
|
||||
|
||||
/// 等待所有受管长连接任务退出。
|
||||
pub async fn wait(&self) {
|
||||
self.connection_tasks.wait().await;
|
||||
}
|
||||
|
||||
/// 是否已进入连接排空阶段。
|
||||
pub fn is_shutting_down(&self) -> bool {
|
||||
self.cancellation.is_cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
/// 将所有 HTTP 响应体接入统一关闭信号。
|
||||
pub async fn drain_response_body(
|
||||
State(connection_drain): State<ConnectionDrain>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
let response = next.run(request).await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let shutdown = connection_drain.cancellation.clone().cancelled_owned();
|
||||
let body = Body::from_stream(body.into_data_stream().take_until(shutdown));
|
||||
Response::from_parts(parts, body)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! HTTP 中间件 —— 连接排空、请求 ID 与 Tracing。
|
||||
|
||||
pub mod connection_drain;
|
||||
pub mod request_id;
|
||||
pub mod trace;
|
||||
+11
-94
@@ -4,7 +4,7 @@ use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use axum::{
|
||||
extract::{connect_info::ConnectInfo, Request, State},
|
||||
extract::{Request, connect_info::ConnectInfo},
|
||||
http::{HeaderMap, HeaderValue},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
@@ -19,7 +19,7 @@ const CF_CONNECTING_IP_HEADER: &str = "cf-connecting-ip";
|
||||
const REAL_IP_HEADER: &str = "x-real-ip";
|
||||
const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for";
|
||||
|
||||
/// 私有 / 回环 / 唯一本地地址网段,用于在自动模式下从 `X-Forwarded-For` 中跳过内网跳板 IP。
|
||||
/// 私有 / 回环 / 唯一本地地址网段,用于从 `X-Forwarded-For` 中跳过内网跳板 IP。
|
||||
static PRIVATE_NETWORKS: LazyLock<Vec<IpNet>> = LazyLock::new(|| {
|
||||
[
|
||||
"10.0.0.0/8",
|
||||
@@ -38,40 +38,6 @@ fn is_private_ip(ip: IpAddr) -> bool {
|
||||
PRIVATE_NETWORKS.iter().any(|network| network.contains(&ip))
|
||||
}
|
||||
|
||||
/// 可信反向代理配置。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TrustedProxyConfig {
|
||||
networks: Vec<IpNet>,
|
||||
}
|
||||
|
||||
impl TrustedProxyConfig {
|
||||
/// 从 IP 或 CIDR 字符串构造可信代理配置。
|
||||
pub fn from_entries(entries: &[String]) -> Result<Self, TrustedProxyConfigError> {
|
||||
let networks = entries
|
||||
.iter()
|
||||
.map(|entry| parse_trusted_proxy_entry(entry))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(Self { networks })
|
||||
}
|
||||
|
||||
fn is_trusted(&self, ip: IpAddr) -> bool {
|
||||
self.networks.iter().any(|network| network.contains(&ip))
|
||||
}
|
||||
|
||||
/// 是否未配置任何可信代理(决定是否启用自动真实 IP 探测)。
|
||||
fn is_empty(&self) -> bool {
|
||||
self.networks.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// 可信代理配置错误。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TrustedProxyConfigError {
|
||||
/// 配置项不是 IP 地址或 CIDR。
|
||||
#[error("invalid trusted proxy entry `{entry}`; expected an IP address or CIDR network")]
|
||||
InvalidEntry { entry: String },
|
||||
}
|
||||
|
||||
/// 请求 ID。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestId(String);
|
||||
@@ -97,7 +63,7 @@ impl RequestId {
|
||||
}
|
||||
}
|
||||
|
||||
/// 由连接来源和可信代理链解析出的客户端 IP。
|
||||
/// 由连接来源和转发头解析出的客户端 IP。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClientIp(String);
|
||||
|
||||
@@ -118,17 +84,7 @@ fn valid_request_id(value: &str) -> bool {
|
||||
|
||||
/// 为请求附加请求 ID,并在响应头中回写。
|
||||
pub async fn attach_request_id(mut request: Request, next: Next) -> Response {
|
||||
attach_request_context(&TrustedProxyConfig::default(), &mut request);
|
||||
run_with_request_id(request, next).await
|
||||
}
|
||||
|
||||
/// 为请求附加请求上下文,并按可信代理配置解析真实客户端 IP。
|
||||
pub async fn attach_request_id_with_proxy_config(
|
||||
State(trusted_proxies): State<TrustedProxyConfig>,
|
||||
mut request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
attach_request_context(&trusted_proxies, &mut request);
|
||||
attach_request_context(&mut request);
|
||||
run_with_request_id(request, next).await
|
||||
}
|
||||
|
||||
@@ -147,17 +103,16 @@ async fn run_with_request_id(mut request: Request, next: Next) -> Response {
|
||||
response
|
||||
}
|
||||
|
||||
fn attach_request_context(trusted_proxies: &TrustedProxyConfig, request: &mut Request) {
|
||||
attach_client_ip_from_connection(trusted_proxies, request);
|
||||
fn attach_request_context(request: &mut Request) {
|
||||
attach_client_ip_from_connection(request);
|
||||
}
|
||||
|
||||
fn attach_client_ip_from_connection(trusted_proxies: &TrustedProxyConfig, request: &mut Request) {
|
||||
fn attach_client_ip_from_connection(request: &mut Request) {
|
||||
let peer_ip = request
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|ConnectInfo(addr)| addr.ip());
|
||||
let client_ip =
|
||||
peer_ip.map(|peer_ip| client_ip_from_peer(peer_ip, request.headers(), trusted_proxies));
|
||||
let client_ip = peer_ip.map(|peer_ip| client_ip_from_peer(peer_ip, request.headers()));
|
||||
|
||||
let Some(client_ip) = client_ip else {
|
||||
return;
|
||||
@@ -165,28 +120,12 @@ fn attach_client_ip_from_connection(trusted_proxies: &TrustedProxyConfig, reques
|
||||
request.extensions_mut().insert(ClientIp(client_ip));
|
||||
}
|
||||
|
||||
fn client_ip_from_peer(
|
||||
peer_ip: IpAddr,
|
||||
headers: &HeaderMap,
|
||||
trusted_proxies: &TrustedProxyConfig,
|
||||
) -> String {
|
||||
if trusted_proxies.is_empty() {
|
||||
// 未配置可信代理:自动模式,按 CF-Connecting-IP → X-Real-IP →
|
||||
// X-Forwarded-For(首个公网 IP)→ peer IP 的优先级解析真实客户端 IP。
|
||||
return auto_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string());
|
||||
}
|
||||
if trusted_proxies.is_trusted(peer_ip) {
|
||||
// 已配置可信代理且直连来源可信:只采信转发头。
|
||||
trusted_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string())
|
||||
} else {
|
||||
// 已配置可信代理但直连来源不可信:忽略转发头,使用 peer IP。
|
||||
peer_ip.to_string()
|
||||
}
|
||||
fn client_ip_from_peer(peer_ip: IpAddr, headers: &HeaderMap) -> String {
|
||||
auto_forwarded_client_ip(headers).unwrap_or_else(|| peer_ip.to_string())
|
||||
}
|
||||
|
||||
/// 自动模式下解析转发头中的真实客户端 IP。
|
||||
/// 解析转发头中的真实客户端 IP。
|
||||
///
|
||||
/// 未配置可信代理时使用,对齐常见反代(Cloudflare / Nginx)默认行为:
|
||||
/// `CF-Connecting-IP` → `X-Real-IP` → `X-Forwarded-For` 首个公网 IP。
|
||||
fn auto_forwarded_client_ip(headers: &HeaderMap) -> Option<String> {
|
||||
if let Some(cf_ip) = header_string(headers, CF_CONNECTING_IP_HEADER) {
|
||||
@@ -212,12 +151,6 @@ fn forwarded_for_public_ip(headers: &HeaderMap) -> Option<String> {
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn trusted_forwarded_client_ip(headers: &HeaderMap) -> Option<String> {
|
||||
header_string(headers, CF_CONNECTING_IP_HEADER)
|
||||
.or_else(|| first_forwarded_for_value(headers))
|
||||
.or_else(|| header_string(headers, REAL_IP_HEADER))
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
@@ -236,19 +169,3 @@ fn first_forwarded_for_value(headers: &HeaderMap) -> Option<String> {
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn parse_trusted_proxy_entry(entry: &str) -> Result<IpNet, TrustedProxyConfigError> {
|
||||
if let Ok(network) = entry.parse::<IpNet>() {
|
||||
return Ok(network);
|
||||
}
|
||||
entry
|
||||
.parse::<IpAddr>()
|
||||
.map(IpNet::from)
|
||||
.map_err(|_| invalid_trusted_proxy_entry(entry))
|
||||
}
|
||||
|
||||
fn invalid_trusted_proxy_entry(entry: &str) -> TrustedProxyConfigError {
|
||||
TrustedProxyConfigError::InvalidEntry {
|
||||
entry: entry.to_string(),
|
||||
}
|
||||
}
|
||||
@@ -10,9 +10,9 @@ use tower_http::{
|
||||
classify::{ServerErrorsAsFailures, ServerErrorsFailureClass, SharedClassifier},
|
||||
trace::TraceLayer,
|
||||
};
|
||||
use tracing::{info, warn, Span};
|
||||
use tracing::{Span, debug, info, warn};
|
||||
|
||||
use crate::http::middleware::request_id::RequestId;
|
||||
use crate::api::middleware::request_id::{ClientIp, RequestId};
|
||||
|
||||
/// HTTP tracing layer type.
|
||||
pub type HttpTraceLayer = TraceLayer<
|
||||
@@ -36,27 +36,25 @@ pub fn http_trace_layer() -> HttpTraceLayer {
|
||||
|
||||
fn make_http_span(request: &Request<Body>) -> Span {
|
||||
let request_id = request_id(request);
|
||||
let client_ip = client_ip(request);
|
||||
let uri = sanitized_uri(request);
|
||||
tracing::info_span!(
|
||||
"http",
|
||||
request_id = request_id.as_deref(),
|
||||
client_ip,
|
||||
method = %request.method(),
|
||||
uri = %uri
|
||||
)
|
||||
}
|
||||
|
||||
fn on_http_request(request: &Request<Body>, _span: &Span) {
|
||||
let request_id = request_id(request);
|
||||
let uri = sanitized_uri(request);
|
||||
info!(
|
||||
request_id = request_id.as_deref(),
|
||||
method = %request.method(),
|
||||
uri = %uri,
|
||||
"received HTTP request"
|
||||
);
|
||||
fn on_http_request(_request: &Request<Body>, _span: &Span) {
|
||||
debug!("received HTTP request");
|
||||
}
|
||||
|
||||
fn on_http_response(response: &Response<Body>, latency: Duration, _span: &Span) {
|
||||
if response.status().is_server_error() {
|
||||
return;
|
||||
}
|
||||
info!(
|
||||
status = response.status().as_u16(),
|
||||
latency_ms = latency.as_millis(),
|
||||
@@ -83,6 +81,10 @@ fn request_id(request: &Request<Body>) -> Option<String> {
|
||||
.map(|rid| rid.as_str().to_string())
|
||||
}
|
||||
|
||||
fn client_ip(request: &Request<Body>) -> Option<&str> {
|
||||
request.extensions().get::<ClientIp>().map(ClientIp::as_str)
|
||||
}
|
||||
|
||||
fn sanitized_uri(request: &Request<Body>) -> String {
|
||||
let uri = request.uri();
|
||||
let path = uri.path();
|
||||
@@ -0,0 +1,9 @@
|
||||
//! 入站 HTTP API 与静态资源。
|
||||
|
||||
pub mod admin;
|
||||
pub mod assets;
|
||||
pub mod client;
|
||||
pub mod middleware;
|
||||
pub mod router;
|
||||
|
||||
pub use router::AppState;
|
||||
@@ -0,0 +1,103 @@
|
||||
//! 顶层 HTTP 路由 —— 组合 OpenAI API、管理端 API 和静态资源服务。
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use axum::{Router, extract::State, http::StatusCode, middleware, routing::get};
|
||||
|
||||
use crate::{
|
||||
api::{
|
||||
admin, assets, client,
|
||||
middleware::{request_id::attach_request_id, trace::http_trace_layer},
|
||||
},
|
||||
auth::service::SessionService,
|
||||
dispatch::{affinity::SessionAffinityService, service::ResponseDispatchService},
|
||||
fleet::{
|
||||
manage::AccountManageService, pool::AccountPoolService, refresh::TokenRefreshService,
|
||||
store::AccountStore,
|
||||
},
|
||||
keys::{manage::KeyManageService, service::KeyVerifier},
|
||||
models::service::ModelService,
|
||||
settings::service::SettingsService,
|
||||
telemetry::{
|
||||
account_usage::query::AccountUsageQueryService, ops::query::OpsQueryService,
|
||||
usage::query::UsageQueryService,
|
||||
},
|
||||
update::service::SystemUpdateService,
|
||||
upstream::openai::{fingerprint::RuntimeFingerprint, token_client::OpenAiTokenClient},
|
||||
};
|
||||
|
||||
/// HTTP API 所需的领域服务集合。
|
||||
#[derive(Clone)]
|
||||
pub struct ApiServices {
|
||||
pub health_probe: Arc<dyn HealthProbe>,
|
||||
pub models: Arc<ModelService>,
|
||||
pub accounts: Arc<dyn AccountStore>,
|
||||
pub client_keys: Arc<KeyVerifier>,
|
||||
pub admin_client_keys: Arc<KeyManageService>,
|
||||
pub admin_sessions: Arc<SessionService>,
|
||||
pub settings: Arc<SettingsService>,
|
||||
pub admin_accounts: Arc<AccountManageService>,
|
||||
pub usage_records: Arc<UsageQueryService>,
|
||||
pub ops_errors: Arc<OpsQueryService>,
|
||||
pub usage: Arc<AccountUsageQueryService>,
|
||||
pub account_pool: Arc<AccountPoolService>,
|
||||
pub token_refresh: Arc<TokenRefreshService<OpenAiTokenClient>>,
|
||||
pub responses: Arc<ResponseDispatchService>,
|
||||
pub session_affinity: Arc<SessionAffinityService>,
|
||||
pub fingerprint: RuntimeFingerprint,
|
||||
pub process_control: Arc<dyn ProcessControl>,
|
||||
pub system_update: Arc<SystemUpdateService>,
|
||||
pub connection_drain: crate::api::middleware::connection_drain::ConnectionDrain,
|
||||
}
|
||||
|
||||
/// 由 bootstrap 实现的进程生命周期控制端口。
|
||||
pub trait ProcessControl: Send + Sync + 'static {
|
||||
fn request_shutdown(&self);
|
||||
fn request_restart(&self, executable_path: PathBuf);
|
||||
}
|
||||
|
||||
/// 由 bootstrap 实现的基础设施健康检查端口。
|
||||
#[async_trait]
|
||||
pub trait HealthProbe: Send + Sync + 'static {
|
||||
async fn check(&self) -> Result<(), String>;
|
||||
}
|
||||
|
||||
/// HTTP handler 共享状态。
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub services: ApiServices,
|
||||
}
|
||||
|
||||
/// 默认前端构建产物目录。
|
||||
pub const DEFAULT_ASSET_DIST_DIR: &str = "web/dist";
|
||||
|
||||
/// 构造整个 HTTP 服务路由。
|
||||
pub fn router() -> Router<AppState> {
|
||||
router_with_assets(DEFAULT_ASSET_DIST_DIR)
|
||||
}
|
||||
|
||||
/// 使用指定前端构建产物目录构造整个 HTTP 服务路由。
|
||||
pub fn router_with_assets(dist_dir: impl AsRef<Path>) -> Router<AppState> {
|
||||
let traced_routes = Router::new()
|
||||
.merge(client::router::router())
|
||||
.merge(admin::router::router())
|
||||
.fallback_service(assets::spa_router(dist_dir))
|
||||
.layer(http_trace_layer());
|
||||
|
||||
Router::new()
|
||||
.route("/healthz", get(healthz))
|
||||
.merge(traced_routes)
|
||||
.layer(middleware::from_fn(attach_request_id))
|
||||
}
|
||||
|
||||
async fn healthz(State(state): State<AppState>) -> StatusCode {
|
||||
if let Err(error) = state.services.health_probe.check().await {
|
||||
tracing::warn!(error, "health check failed");
|
||||
return StatusCode::SERVICE_UNAVAILABLE;
|
||||
}
|
||||
StatusCode::NO_CONTENT
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! 管理员认证与 Redis 会话。
|
||||
|
||||
pub mod service;
|
||||
pub mod store;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,199 @@
|
||||
//! 管理员认证领域服务。
|
||||
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::infra::identity::{
|
||||
generate_admin_session_token, hash_admin_password, verify_admin_password,
|
||||
};
|
||||
|
||||
use super::{
|
||||
store::{PgAdminUserStore, RedisAdminSessionStore},
|
||||
types::{LoginSession, SessionError},
|
||||
};
|
||||
|
||||
const LOGIN_FAILURE_WINDOW: Duration = Duration::minutes(15);
|
||||
const LOGIN_LOCK_DURATION: Duration = Duration::minutes(15);
|
||||
const LOGIN_MAX_FAILURES: u32 = 5;
|
||||
const LOGIN_FAILURE_SOURCE_LIMIT: usize = 10_000;
|
||||
|
||||
/// 管理员会话服务。
|
||||
#[derive(Clone)]
|
||||
pub struct SessionService {
|
||||
users: PgAdminUserStore,
|
||||
sessions: RedisAdminSessionStore,
|
||||
default_username: String,
|
||||
session_ttl_minutes: u64,
|
||||
login_failures: Arc<Mutex<HashMap<String, LoginFailureState>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct LoginFailureState {
|
||||
failures: u32,
|
||||
first_failed_at: DateTime<Utc>,
|
||||
locked_until: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl SessionService {
|
||||
/// 构造服务。
|
||||
pub fn new(
|
||||
users: PgAdminUserStore,
|
||||
sessions: RedisAdminSessionStore,
|
||||
default_username: String,
|
||||
session_ttl_minutes: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
users,
|
||||
sessions,
|
||||
default_username,
|
||||
session_ttl_minutes,
|
||||
login_failures: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验管理员会话。
|
||||
pub async fn validate(&self, session_id: Option<&str>) -> Result<bool, SessionError> {
|
||||
let Some(session_id) = session_id else {
|
||||
return Ok(false);
|
||||
};
|
||||
self.sessions
|
||||
.validate_session(session_id)
|
||||
.await
|
||||
.map_err(|_| SessionError::Validate)
|
||||
}
|
||||
|
||||
/// 确保默认管理员存在。
|
||||
pub async fn ensure_default_admin(&self, password: &str) -> Result<bool, SessionError> {
|
||||
let password_hash =
|
||||
hash_admin_password(password).map_err(|_| SessionError::HashPassword)?;
|
||||
self.users
|
||||
.ensure_default_admin(&password_hash)
|
||||
.await
|
||||
.map_err(|_| SessionError::CreateAdmin)
|
||||
}
|
||||
|
||||
/// 登录并创建管理员会话。
|
||||
pub async fn login(
|
||||
&self,
|
||||
source: &str,
|
||||
username: Option<&str>,
|
||||
password: &str,
|
||||
) -> Result<Option<LoginSession>, SessionError> {
|
||||
let source = login_source_key(source);
|
||||
self.ensure_login_allowed(&source).await?;
|
||||
|
||||
let username = username.unwrap_or(&self.default_username);
|
||||
if username != self.default_username.as_str() {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(admin) = self
|
||||
.users
|
||||
.load_first_admin()
|
||||
.await
|
||||
.map_err(|_| SessionError::LoadAdmin)?
|
||||
else {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
};
|
||||
let password_matches = verify_admin_password(password, &admin.password_hash)
|
||||
.map_err(|_| SessionError::VerifyPassword)?;
|
||||
if !password_matches {
|
||||
self.record_failed_login(&source).await;
|
||||
return Ok(None);
|
||||
}
|
||||
self.clear_failed_login(&source).await;
|
||||
let session_id = generate_admin_session_token();
|
||||
let ttl_minutes = self.session_ttl_minutes.min(i64::MAX as u64) as i64;
|
||||
let expires_at = Utc::now() + Duration::minutes(ttl_minutes);
|
||||
self.sessions
|
||||
.create_session(
|
||||
&session_id,
|
||||
&admin.id,
|
||||
Duration::minutes(ttl_minutes.max(1)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| SessionError::CreateSession)?;
|
||||
Ok(Some(LoginSession {
|
||||
session_id,
|
||||
expires_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 删除管理员会话。
|
||||
pub async fn delete_session(&self, session_id: &str) -> Result<bool, SessionError> {
|
||||
self.sessions
|
||||
.delete_session(session_id)
|
||||
.await
|
||||
.map_err(|_| SessionError::DeleteSession)
|
||||
}
|
||||
|
||||
async fn ensure_login_allowed(&self, source: &str) -> Result<(), SessionError> {
|
||||
let now = Utc::now();
|
||||
let login_locked = {
|
||||
let mut failures = self.login_failures.lock().await;
|
||||
retain_active_login_failures(&mut failures, now);
|
||||
if !failures.contains_key(source) && failures.len() >= LOGIN_FAILURE_SOURCE_LIMIT {
|
||||
return Err(SessionError::LoginThrottled);
|
||||
}
|
||||
failures
|
||||
.get(source)
|
||||
.and_then(|state| state.locked_until)
|
||||
.is_some_and(|locked_until| locked_until > now)
|
||||
};
|
||||
if login_locked {
|
||||
return Err(SessionError::LoginThrottled);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_failed_login(&self, source: &str) {
|
||||
let now = Utc::now();
|
||||
let mut failures = self.login_failures.lock().await;
|
||||
retain_active_login_failures(&mut failures, now);
|
||||
let state = failures
|
||||
.entry(source.to_string())
|
||||
.or_insert_with(|| LoginFailureState {
|
||||
failures: 0,
|
||||
first_failed_at: now,
|
||||
locked_until: None,
|
||||
});
|
||||
if now - state.first_failed_at > LOGIN_FAILURE_WINDOW {
|
||||
state.failures = 0;
|
||||
state.first_failed_at = now;
|
||||
state.locked_until = None;
|
||||
}
|
||||
state.failures += 1;
|
||||
if state.failures >= LOGIN_MAX_FAILURES {
|
||||
state.locked_until = Some(now + LOGIN_LOCK_DURATION);
|
||||
}
|
||||
drop(failures);
|
||||
}
|
||||
|
||||
async fn clear_failed_login(&self, source: &str) {
|
||||
self.login_failures.lock().await.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
fn login_source_key(source: &str) -> String {
|
||||
let source = source.trim();
|
||||
if source.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
source.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn retain_active_login_failures(
|
||||
failures: &mut HashMap<String, LoginFailureState>,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
failures.retain(|_, state| {
|
||||
state
|
||||
.locked_until
|
||||
.is_some_and(|locked_until| locked_until > now)
|
||||
|| now - state.first_failed_at <= LOGIN_FAILURE_WINDOW
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
//! 管理员用户与会话存储。
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use redis::AsyncCommands;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::PgPool;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::infra::{identity::hash_credential, redis::RedisConnection};
|
||||
|
||||
/// 已持久化的管理员用户。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StoredAdminUser {
|
||||
/// 管理员用户 ID。
|
||||
pub id: String,
|
||||
/// 管理员密码哈希。
|
||||
pub password_hash: String,
|
||||
}
|
||||
|
||||
/// 管理认证存储错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AuthStoreError {
|
||||
#[error("PostgreSQL admin user operation failed: {0}")]
|
||||
Postgres(#[from] sqlx::Error),
|
||||
#[error("Redis admin session operation failed: {0}")]
|
||||
Redis(#[from] redis::RedisError),
|
||||
#[error("invalid Redis admin session value: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
pub type AuthStoreResult<T> = Result<T, AuthStoreError>;
|
||||
|
||||
/// PostgreSQL 管理员用户存储。
|
||||
#[derive(Clone)]
|
||||
pub struct PgAdminUserStore {
|
||||
pool: PgPool,
|
||||
}
|
||||
|
||||
impl PgAdminUserStore {
|
||||
/// 构造存储。
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// 如果还没有管理员用户,则创建默认管理员。
|
||||
pub async fn ensure_default_admin(&self, password_hash: &str) -> AuthStoreResult<bool> {
|
||||
let now = Utc::now();
|
||||
let result = sqlx::query(
|
||||
r"
|
||||
insert into admin_users (id, password_hash, created_at, updated_at)
|
||||
select $1, $2, $3, $3
|
||||
where not exists (select 1 from admin_users)
|
||||
on conflict do nothing",
|
||||
)
|
||||
.bind("admin_1")
|
||||
.bind(password_hash)
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
/// 读取第一个管理员用户。
|
||||
pub async fn load_first_admin(&self) -> AuthStoreResult<Option<StoredAdminUser>> {
|
||||
let row = sqlx::query_as::<_, (String, String)>(
|
||||
"select id, password_hash from admin_users order by created_at asc, id asc limit 1",
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(row.map(|(id, password_hash)| StoredAdminUser { id, password_hash }))
|
||||
}
|
||||
}
|
||||
|
||||
/// Redis 管理员会话存储。
|
||||
#[derive(Clone)]
|
||||
pub struct RedisAdminSessionStore {
|
||||
redis: RedisConnection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct StoredAdminSession {
|
||||
user_id: String,
|
||||
created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl RedisAdminSessionStore {
|
||||
pub fn new(redis: RedisConnection) -> Self {
|
||||
Self { redis }
|
||||
}
|
||||
|
||||
pub async fn create_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
user_id: &str,
|
||||
ttl: Duration,
|
||||
) -> AuthStoreResult<()> {
|
||||
let key = self.session_key(session_id);
|
||||
let value = serde_json::to_string(&StoredAdminSession {
|
||||
user_id: user_id.to_string(),
|
||||
created_at: Utc::now(),
|
||||
})?;
|
||||
let ttl_seconds = ttl.num_seconds().max(1) as u64;
|
||||
let mut connection = self.redis.manager();
|
||||
let _: () = redis::cmd("SET")
|
||||
.arg(key)
|
||||
.arg(value)
|
||||
.arg("EX")
|
||||
.arg(ttl_seconds)
|
||||
.query_async(&mut connection)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn validate_session(&self, session_id: &str) -> AuthStoreResult<bool> {
|
||||
let mut connection = self.redis.manager();
|
||||
let value: Option<String> = connection.get(self.session_key(session_id)).await?;
|
||||
let Some(value) = value else {
|
||||
return Ok(false);
|
||||
};
|
||||
let session: StoredAdminSession = serde_json::from_str(&value)?;
|
||||
Ok(!session.user_id.trim().is_empty())
|
||||
}
|
||||
|
||||
pub async fn delete_session(&self, session_id: &str) -> AuthStoreResult<bool> {
|
||||
let mut connection = self.redis.manager();
|
||||
let deleted: u64 = connection.del(self.session_key(session_id)).await?;
|
||||
Ok(deleted > 0)
|
||||
}
|
||||
|
||||
fn session_key(&self, session_id: &str) -> String {
|
||||
self.redis
|
||||
.key(&format!("admin:session:{}", hash_credential(session_id)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//! 管理员认证领域类型。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use thiserror::Error;
|
||||
|
||||
/// 管理员登录成功后的会话。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LoginSession {
|
||||
/// 会话 ID。
|
||||
pub session_id: String,
|
||||
/// 过期时间。
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 管理员会话错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SessionError {
|
||||
/// 校验管理员会话失败。
|
||||
#[error("failed to validate admin session")]
|
||||
Validate,
|
||||
/// 计算管理员密码哈希失败。
|
||||
#[error("failed to hash admin password")]
|
||||
HashPassword,
|
||||
/// 创建默认管理员失败。
|
||||
#[error("failed to create default admin user")]
|
||||
CreateAdmin,
|
||||
/// 读取管理员用户失败。
|
||||
#[error("failed to load admin user")]
|
||||
LoadAdmin,
|
||||
/// 校验管理员密码失败。
|
||||
#[error("failed to verify admin password")]
|
||||
VerifyPassword,
|
||||
/// 创建管理员会话失败。
|
||||
#[error("failed to create admin session")]
|
||||
CreateSession,
|
||||
/// 删除管理员会话失败。
|
||||
#[error("failed to delete admin session")]
|
||||
DeleteSession,
|
||||
/// 管理员登录尝试过于频繁。
|
||||
#[error("too many admin login attempts")]
|
||||
LoginThrottled,
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::{collections::BTreeMap, env, path::Path};
|
||||
|
||||
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use url::Url;
|
||||
|
||||
const WEAK_ADMIN_PASSWORDS: &[&str] = &[
|
||||
"",
|
||||
@@ -24,9 +26,6 @@ pub struct AppConfig {
|
||||
/// 运行时模型别名,由数据库设置加载,配置文件不承载该字段。
|
||||
#[serde(skip)]
|
||||
pub model_aliases: BTreeMap<String, String>,
|
||||
/// 运行时模型账号路由,由 `model_account_routes` 表加载,配置文件不承载该字段。
|
||||
#[serde(skip)]
|
||||
pub model_account_routes: BTreeMap<String, Vec<String>>,
|
||||
/// 认证与刷新配置,由固定默认值和数据库运行时设置承载,配置文件不承载该字段。
|
||||
#[serde(skip)]
|
||||
pub auth: AuthConfig,
|
||||
@@ -35,12 +34,14 @@ pub struct AppConfig {
|
||||
pub quota: QuotaConfig,
|
||||
/// 数据库配置。
|
||||
pub database: DatabaseConfig,
|
||||
/// Redis 运行态存储配置。
|
||||
pub redis: RedisConfig,
|
||||
/// TLS 偏好配置。
|
||||
#[serde(default)]
|
||||
pub tls: TlsConfig,
|
||||
/// WebSocket 连接池配置。
|
||||
/// WebSocket 连接池启动设置。
|
||||
#[serde(default)]
|
||||
pub ws_pool: WebSocketPoolConfig,
|
||||
pub ws_pool: WebSocketPoolSettings,
|
||||
/// 上游请求指纹默认配置。
|
||||
#[serde(default)]
|
||||
pub fingerprint: FingerprintConfig,
|
||||
@@ -50,6 +51,9 @@ pub struct AppConfig {
|
||||
/// 日志配置。
|
||||
#[serde(default)]
|
||||
pub logging: LoggingConfig,
|
||||
/// PostgreSQL 遥测事实记录配置。
|
||||
#[serde(default)]
|
||||
pub telemetry: TelemetryConfig,
|
||||
}
|
||||
|
||||
/// HTTP 监听配置。
|
||||
@@ -60,9 +64,6 @@ pub struct ServerConfig {
|
||||
pub host: String,
|
||||
/// 监听端口。
|
||||
pub port: u16,
|
||||
/// 可信反向代理 IP 或 CIDR。只有这些 peer 的转发 IP 头会被采信。
|
||||
#[serde(default)]
|
||||
pub trusted_proxies: Vec<String>,
|
||||
}
|
||||
|
||||
/// 上游 API 地址配置。
|
||||
@@ -98,12 +99,12 @@ pub struct AuthConfig {
|
||||
impl Default for AuthConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_margin_seconds: 3600,
|
||||
refresh_enabled: true,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
rotation_strategy: "least_used".to_string(),
|
||||
rotation_strategy: "smart".to_string(),
|
||||
tier_priority: Vec::new(),
|
||||
oauth_client_id: "app_EMoamEEZ73f0CkXaXp7hrann".to_string(),
|
||||
oauth_token_endpoint: "https://auth.openai.com/oauth/token".to_string(),
|
||||
@@ -143,7 +144,15 @@ fn default_skip_exhausted() -> bool {
|
||||
/// 数据库连接配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DatabaseConfig {
|
||||
/// SQLite 连接 URL。
|
||||
/// PostgreSQL 连接 URL。
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
/// Redis 连接配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RedisConfig {
|
||||
/// Redis 连接 URL。
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
@@ -154,24 +163,36 @@ pub struct TlsConfig {
|
||||
pub force_http11: bool,
|
||||
}
|
||||
|
||||
/// WebSocket 连接池配置。
|
||||
/// WebSocket 连接池启动设置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WebSocketPoolConfig {
|
||||
pub struct WebSocketPoolSettings {
|
||||
/// 是否启用连接池。
|
||||
pub enabled: bool,
|
||||
/// 连接最大存活时长。
|
||||
pub max_age_ms: u64,
|
||||
/// 单账号最大连接数。
|
||||
pub max_per_account: usize,
|
||||
/// 首个真实输出(首事件)到达前的绝对超时;`0` 表示禁用。
|
||||
///
|
||||
/// 从发出 `response.create` 起算,覆盖建连/发送/上游排队/思考的全程,直到收到
|
||||
/// 首个真实内容帧(`response.created`/`response.in_progress` 等纯生命周期帧不计)。
|
||||
/// 超时判定为连接落到病态上游后端,丢弃并换新连接重试。
|
||||
#[serde(default = "default_ws_initial_event_timeout_ms")]
|
||||
pub initial_event_timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for WebSocketPoolConfig {
|
||||
fn default_ws_initial_event_timeout_ms() -> u64 {
|
||||
20_000
|
||||
}
|
||||
|
||||
impl Default for WebSocketPoolSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
max_age_ms: 3_300_000,
|
||||
max_per_account: 8,
|
||||
initial_event_timeout_ms: default_ws_initial_event_timeout_ms(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,12 +287,10 @@ impl FingerprintHeaderConfig {
|
||||
|
||||
/// 管理员初始化与会话配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AdminConfig {
|
||||
/// 会话有效期(分钟)。
|
||||
pub session_ttl_minutes: u64,
|
||||
/// 会话清理周期(秒)。
|
||||
#[serde(default = "default_session_cleanup_interval")]
|
||||
pub session_cleanup_interval_secs: u64,
|
||||
/// 默认管理员用户名(首次启动时创建)
|
||||
#[serde(default = "default_admin_username")]
|
||||
pub default_username: String,
|
||||
@@ -284,7 +303,6 @@ impl Default for AdminConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
session_ttl_minutes: 1440,
|
||||
session_cleanup_interval_secs: default_session_cleanup_interval(),
|
||||
default_username: default_admin_username(),
|
||||
default_password: default_admin_password(),
|
||||
}
|
||||
@@ -315,10 +333,6 @@ pub enum AdminConfigError {
|
||||
WeakDefaultPassword,
|
||||
}
|
||||
|
||||
fn default_session_cleanup_interval() -> u64 {
|
||||
3600 // 默认每小时清理一次过期会话
|
||||
}
|
||||
|
||||
fn default_admin_username() -> String {
|
||||
"admin".to_string()
|
||||
}
|
||||
@@ -327,24 +341,142 @@ fn default_admin_password() -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// 日志持久化配置。
|
||||
/// 应用结构化日志配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct LoggingConfig {
|
||||
/// 日志目录。
|
||||
pub directory: String,
|
||||
/// 保留天数。
|
||||
pub retention_days: usize,
|
||||
/// 是否启用日志。
|
||||
pub enabled: bool,
|
||||
/// 默认日志级别;`RUST_LOG` 存在时优先使用环境变量。
|
||||
pub level: String,
|
||||
/// 是否写入标准输出。
|
||||
pub stdout: bool,
|
||||
/// 文件日志配置。
|
||||
pub file: FileLoggingConfig,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
directory: ".runtime/logs".to_string(),
|
||||
retention_days: 14,
|
||||
enabled: true,
|
||||
level: "info".to_string(),
|
||||
stdout: true,
|
||||
file: FileLoggingConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 应用结构化文件日志配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FileLoggingConfig {
|
||||
/// 是否写入轮转文件。
|
||||
pub enabled: bool,
|
||||
/// 日志目录。
|
||||
pub directory: String,
|
||||
/// 按中国自然日计算的保留天数。
|
||||
pub retention_days: usize,
|
||||
/// 单文件大小上限,单位 MiB。
|
||||
pub max_file_size_mb: u64,
|
||||
/// 文件总数上限。
|
||||
pub max_files: usize,
|
||||
}
|
||||
|
||||
impl Default for FileLoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
directory: ".runtime/logs".to_string(),
|
||||
retention_days: 14,
|
||||
max_file_size_mb: 20,
|
||||
max_files: 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// PostgreSQL 遥测事实记录配置。
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TelemetryConfig {
|
||||
/// 是否记录成功与失败代理事实。
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for TelemetryConfig {
|
||||
fn default() -> Self {
|
||||
Self { enabled: true }
|
||||
}
|
||||
}
|
||||
|
||||
const CONFIG_FILE_ENV: &str = "CPR_CONFIG_FILE";
|
||||
const ADMIN_DEFAULT_PASSWORD_ENV: &str = "CPR_ADMIN_DEFAULT_PASSWORD";
|
||||
const POSTGRES_PASSWORD_ENV: &str = "CPR_POSTGRES_PASSWORD";
|
||||
const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD";
|
||||
|
||||
impl AppConfig {
|
||||
/// 从 `CPR_CONFIG_FILE` 或当前目录 `config.yaml` 加载配置文件,并应用运行时密码。
|
||||
pub fn load() -> Result<Self, ::config::ConfigError> {
|
||||
let config_file = env::var(CONFIG_FILE_ENV)
|
||||
.ok()
|
||||
.filter(|value| !value.trim().is_empty());
|
||||
match config_file {
|
||||
Some(path) => load_file(path),
|
||||
None => Self::load_from_dir("."),
|
||||
}
|
||||
}
|
||||
|
||||
/// 从指定目录加载配置文件,并应用运行时密码。
|
||||
pub fn load_from_dir(config_dir: impl AsRef<Path>) -> Result<Self, ::config::ConfigError> {
|
||||
load_file(config_dir.as_ref().join("config.yaml"))
|
||||
}
|
||||
}
|
||||
|
||||
fn load_file(config_file: impl AsRef<Path>) -> Result<AppConfig, ::config::ConfigError> {
|
||||
let mut app_config: AppConfig = ::config::Config::builder()
|
||||
.add_source(::config::File::from(config_file.as_ref()).required(true))
|
||||
.build()?
|
||||
.try_deserialize()?;
|
||||
apply_password_override(
|
||||
&mut app_config.database.url,
|
||||
"database.url",
|
||||
POSTGRES_PASSWORD_ENV,
|
||||
)?;
|
||||
apply_password_override(&mut app_config.redis.url, "redis.url", REDIS_PASSWORD_ENV)?;
|
||||
if let Some(password) = password_override(ADMIN_DEFAULT_PASSWORD_ENV)? {
|
||||
app_config.admin.default_password = password;
|
||||
}
|
||||
Ok(app_config)
|
||||
}
|
||||
|
||||
fn apply_password_override(
|
||||
raw_url: &mut String,
|
||||
config_key: &str,
|
||||
env_name: &str,
|
||||
) -> Result<(), ::config::ConfigError> {
|
||||
let Some(password) = password_override(env_name)? else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut url = Url::parse(raw_url).map_err(|_| {
|
||||
::config::ConfigError::Message(format!(
|
||||
"{config_key} must be a valid URL when {env_name} is set"
|
||||
))
|
||||
})?;
|
||||
let encoded_password = utf8_percent_encode(&password, NON_ALPHANUMERIC).to_string();
|
||||
url.set_password(Some(&encoded_password)).map_err(|()| {
|
||||
::config::ConfigError::Message(format!(
|
||||
"{config_key} does not support a password override from {env_name}"
|
||||
))
|
||||
})?;
|
||||
*raw_url = url.into();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn password_override(env_name: &str) -> Result<Option<String>, ::config::ConfigError> {
|
||||
match env::var(env_name) {
|
||||
Ok(password) if password.is_empty() => Err(::config::ConfigError::Message(format!(
|
||||
"{env_name} must not be empty"
|
||||
))),
|
||||
Ok(password) => Ok(Some(password)),
|
||||
Err(env::VarError::NotPresent) => Ok(None),
|
||||
Err(env::VarError::NotUnicode(_)) => Err(::config::ConfigError::Message(format!(
|
||||
"{env_name} must contain valid UTF-8"
|
||||
))),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//! 进程装配、共享状态与后台任务。
|
||||
|
||||
pub mod config;
|
||||
pub mod services;
|
||||
pub mod shutdown;
|
||||
pub mod state;
|
||||
pub mod tasks;
|
||||
@@ -0,0 +1,679 @@
|
||||
//! 运行时服务集合。
|
||||
|
||||
use std::{env, error::Error, future::IntoFuture, sync::Arc, time::Duration};
|
||||
|
||||
use axum::Router;
|
||||
|
||||
use crate::{
|
||||
auth::{
|
||||
service::SessionService,
|
||||
store::{PgAdminUserStore, RedisAdminSessionStore},
|
||||
},
|
||||
bootstrap::config::AppConfig,
|
||||
dispatch::{
|
||||
affinity::{RedisSessionAffinityStore, SessionAffinityService},
|
||||
service::{ResponseDispatchService, ResponseDispatchServiceParts},
|
||||
},
|
||||
fleet::{
|
||||
cookies::PgCookieStore,
|
||||
manage::{AccountManageService, AccountManageServiceParts},
|
||||
pool::{AccountPoolOptions, AccountPoolService, AccountPoolStaticSettings},
|
||||
refresh::{
|
||||
RedisRefreshLeaseStore, RefreshPolicy, RuntimeRefreshPolicy, TokenRefreshService,
|
||||
},
|
||||
store::{AccountStore, PgAccountStore},
|
||||
},
|
||||
infra::{
|
||||
database::connect,
|
||||
identity::AccountPseudonymizer,
|
||||
logging::{LogError, LogGuard, RotationConfig, TracingConfig, init_tracing},
|
||||
paths::{ensure_data_dir, load_or_create_identity_secret},
|
||||
redis::RedisConnection,
|
||||
},
|
||||
keys::{manage::KeyManageService, service::KeyVerifier, store::PgClientKeyStore},
|
||||
models::{
|
||||
service::{ModelService, ModelServiceError},
|
||||
store::{ModelSnapshotStore, RedisModelSnapshotStore},
|
||||
types::ModelConfig,
|
||||
},
|
||||
settings::{SettingsSnapshot, service::SettingsService},
|
||||
telemetry::{
|
||||
account_usage::query::AccountUsageQueryService,
|
||||
account_usage::store::{AccountUsageStore, PgAccountUsageStore},
|
||||
buckets::store::PgRequestBucketStore,
|
||||
ops::query::OpsQueryService,
|
||||
ops::store::PgOpsErrorLogStore,
|
||||
recorder::Recorder,
|
||||
usage::query::UsageQueryService,
|
||||
usage::store::{DEFAULT_USAGE_RECORD_CAPTURE_BODY, PgUsageRecordStore},
|
||||
},
|
||||
update::service::SystemUpdateService,
|
||||
upstream::openai::{
|
||||
fingerprint::{Fingerprint, PgFingerprintStore, RuntimeFingerprint},
|
||||
token_client::{OpenAiTokenClient, TokenClientConfig, default_openai_token_client},
|
||||
transport::{
|
||||
CodexBackendClient, CodexModelCatalogClient, CodexWebSocketPool,
|
||||
CodexWebSocketPoolConfig, build_reqwest_client, tls::CustomCaError,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS";
|
||||
|
||||
/// 运行时热路径内存状态初始化错误。
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RuntimeServiceInitializationError {
|
||||
#[error("failed to initialize model catalog runtime cache: {0}")]
|
||||
Models(#[from] ModelServiceError),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// BackgroundTaskStores
|
||||
// ============================================================================
|
||||
|
||||
/// 后台任务需要的具体存储适配器集合。
|
||||
#[derive(Clone)]
|
||||
pub struct BackgroundTaskStores {
|
||||
pub redis: RedisConnection,
|
||||
pub accounts: PgAccountStore,
|
||||
pub admin_users: PgAdminUserStore,
|
||||
pub admin_sessions: RedisAdminSessionStore,
|
||||
pub cookies: PgCookieStore,
|
||||
pub fingerprints: PgFingerprintStore,
|
||||
pub session_affinity: RedisSessionAffinityStore,
|
||||
pub refresh_leases: RedisRefreshLeaseStore,
|
||||
pub model_snapshots: RedisModelSnapshotStore,
|
||||
pub client_keys: PgClientKeyStore,
|
||||
pub usage_records: PgUsageRecordStore,
|
||||
pub ops_errors: PgOpsErrorLogStore,
|
||||
pub account_usage: PgAccountUsageStore,
|
||||
pub request_buckets: PgRequestBucketStore,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Services struct
|
||||
// ============================================================================
|
||||
|
||||
/// 运行时服务集合。
|
||||
#[derive(Clone)]
|
||||
pub struct Services {
|
||||
pub database: sqlx::PgPool,
|
||||
pub redis: RedisConnection,
|
||||
pub models: Arc<ModelService>,
|
||||
pub accounts: Arc<dyn AccountStore>,
|
||||
pub client_keys: Arc<KeyVerifier>,
|
||||
pub admin_client_keys: Arc<KeyManageService>,
|
||||
pub admin_sessions: Arc<SessionService>,
|
||||
pub settings: Arc<SettingsService>,
|
||||
pub refresh_policy: RuntimeRefreshPolicy,
|
||||
pub admin_accounts: Arc<AccountManageService>,
|
||||
pub usage_records: Arc<UsageQueryService>,
|
||||
pub ops_errors: Arc<OpsQueryService>,
|
||||
pub usage: Arc<AccountUsageQueryService>,
|
||||
pub account_pool: Arc<AccountPoolService>,
|
||||
pub token_refresh: Arc<TokenRefreshService<OpenAiTokenClient>>,
|
||||
pub responses: Arc<ResponseDispatchService>,
|
||||
pub session_affinity: Arc<SessionAffinityService>,
|
||||
pub codex: Arc<CodexBackendClient>,
|
||||
pub websocket_pool: Option<Arc<CodexWebSocketPool>>,
|
||||
pub fingerprint: RuntimeFingerprint,
|
||||
pub account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
pub system_update: Arc<SystemUpdateService>,
|
||||
pub connection_drain: crate::api::middleware::connection_drain::ConnectionDrain,
|
||||
pub background_tasks: BackgroundTaskStores,
|
||||
pub(crate) account_pool_static: AccountPoolStaticSettings,
|
||||
}
|
||||
|
||||
/// 使用记录运行选项。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct UsageRecordOptions {
|
||||
pub enabled: bool,
|
||||
pub capture_body: bool,
|
||||
}
|
||||
|
||||
impl UsageRecordOptions {
|
||||
pub fn from_config(config: &AppConfig) -> Self {
|
||||
Self {
|
||||
enabled: config.telemetry.enabled,
|
||||
capture_body: DEFAULT_USAGE_RECORD_CAPTURE_BODY,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Services {
|
||||
pub fn new(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
) -> Self {
|
||||
Self::try_new(config, stores, fingerprint)
|
||||
.expect("failed to build runtime services with configured TLS transport")
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
Self::try_with_usage_record_options(
|
||||
config,
|
||||
stores,
|
||||
fingerprint,
|
||||
UsageRecordOptions::from_config(config),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn try_with_usage_record_options(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
let mut identity_secret = [0u8; 32];
|
||||
rand::Rng::fill_bytes(&mut rand::rng(), &mut identity_secret);
|
||||
Self::try_with_identity_secret_and_usage_record_options(
|
||||
config,
|
||||
stores,
|
||||
fingerprint,
|
||||
identity_secret,
|
||||
usage_record_options,
|
||||
)
|
||||
}
|
||||
|
||||
fn try_with_identity_secret_and_usage_record_options(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
identity_secret: [u8; 32],
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
let account_pseudonymizer = Arc::new(AccountPseudonymizer::new(identity_secret));
|
||||
let account_identity = Arc::new(crate::dispatch::affinity::AccountIdentityService::new(
|
||||
(*account_pseudonymizer).clone(),
|
||||
));
|
||||
let database = stores.accounts.pool().clone();
|
||||
let redis = stores.redis.clone();
|
||||
let account_store = Arc::new(stores.accounts.clone()) as Arc<dyn AccountStore>;
|
||||
let account_usage_store_trait =
|
||||
Arc::new(stores.account_usage.clone()) as Arc<dyn AccountUsageStore>;
|
||||
let websocket_pool = config.ws_pool.enabled.then(|| {
|
||||
Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig {
|
||||
enabled: true,
|
||||
max_age: std::time::Duration::from_millis(config.ws_pool.max_age_ms),
|
||||
max_per_account: config.ws_pool.max_per_account,
|
||||
initial_event_timeout: (config.ws_pool.initial_event_timeout_ms > 0).then(|| {
|
||||
std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms)
|
||||
}),
|
||||
..CodexWebSocketPoolConfig::default()
|
||||
}))
|
||||
});
|
||||
let codex = {
|
||||
let websocket_initial_event_timeout = (config.ws_pool.initial_event_timeout_ms > 0)
|
||||
.then(|| std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms));
|
||||
let client = CodexBackendClient::new(
|
||||
build_reqwest_client(config.tls.force_http11)?,
|
||||
config.api.base_url.clone(),
|
||||
fingerprint.clone(),
|
||||
)
|
||||
.with_websocket_initial_event_timeout(websocket_initial_event_timeout);
|
||||
if let Some(pool) = &websocket_pool {
|
||||
Arc::new(client.with_websocket_pool(pool.clone()))
|
||||
} else {
|
||||
Arc::new(client)
|
||||
}
|
||||
};
|
||||
let usage_records = Arc::new(UsageQueryService::new(stores.usage_records.clone()));
|
||||
let ops_errors = Arc::new(OpsQueryService::new(stores.ops_errors.clone()));
|
||||
let recorder = Arc::new(Recorder::new(
|
||||
stores.usage_records.clone(),
|
||||
stores.ops_errors.clone(),
|
||||
usage_record_options.enabled,
|
||||
usage_record_options.capture_body,
|
||||
));
|
||||
let settings_snapshot = settings_snapshot_from_config(config);
|
||||
let account_pool_options = account_pool_options_from_config(config);
|
||||
let account_pool_static = AccountPoolStaticSettings {
|
||||
skip_quota_limited: account_pool_options.skip_quota_limited,
|
||||
tier_priority: account_pool_options.tier_priority.clone(),
|
||||
};
|
||||
let account_pool = Arc::new(AccountPoolService::new(
|
||||
account_store.clone(),
|
||||
account_usage_store_trait,
|
||||
account_pool_options,
|
||||
settings_snapshot.request_interval_ms,
|
||||
));
|
||||
let refresh_policy = RuntimeRefreshPolicy::new(RefreshPolicy {
|
||||
refresh_margin_seconds: config.auth.refresh_margin_seconds,
|
||||
refresh_concurrency: config.auth.refresh_concurrency,
|
||||
});
|
||||
let settings = Arc::new(SettingsService::new(
|
||||
settings_snapshot,
|
||||
stores.accounts.pool().clone(),
|
||||
));
|
||||
let admin_sessions = Arc::new(SessionService::new(
|
||||
stores.admin_users.clone(),
|
||||
stores.admin_sessions.clone(),
|
||||
config.admin.default_username.clone(),
|
||||
config.admin.session_ttl_minutes,
|
||||
));
|
||||
let admin_client_keys = Arc::new(KeyManageService::new(stores.client_keys.clone()));
|
||||
let client_keys = Arc::new(KeyVerifier::new(stores.client_keys.clone()));
|
||||
let token_client = default_openai_token_client(token_client_config(config));
|
||||
let token_refresh = Arc::new(
|
||||
TokenRefreshService::new(
|
||||
stores.accounts.clone(),
|
||||
refresh_policy.clone(),
|
||||
token_client.clone(),
|
||||
)
|
||||
.with_refresh_lease_store(stores.refresh_leases.clone()),
|
||||
);
|
||||
let upstream_client: Arc<dyn CodexModelCatalogClient> = codex.clone();
|
||||
let store: Arc<dyn ModelSnapshotStore> = Arc::new(stores.model_snapshots.clone());
|
||||
let models = Arc::new(ModelService::new(
|
||||
ModelConfig {
|
||||
model_aliases: config.model_aliases.clone(),
|
||||
},
|
||||
Some(store),
|
||||
Some(upstream_client),
|
||||
));
|
||||
let admin_accounts = Arc::new(AccountManageService::new(AccountManageServiceParts {
|
||||
store: stores.accounts.clone(),
|
||||
cookies: stores.cookies.clone(),
|
||||
codex: codex.clone(),
|
||||
models: models.clone(),
|
||||
account_pool: account_pool.clone(),
|
||||
token_refresher: Arc::new(token_client),
|
||||
refresh_leases: stores.refresh_leases.clone(),
|
||||
oauth: crate::fleet::manage::oauth::AccountOAuthService::new(
|
||||
reqwest::Client::new(),
|
||||
config.auth.oauth_client_id.clone(),
|
||||
config.auth.oauth_token_endpoint.clone(),
|
||||
),
|
||||
refresh_policy: refresh_policy.clone(),
|
||||
account_pseudonymizer: account_pseudonymizer.clone(),
|
||||
}));
|
||||
let usage = Arc::new(AccountUsageQueryService::new(stores.account_usage.clone()));
|
||||
let session_affinity =
|
||||
Arc::new(SessionAffinityService::new(stores.session_affinity.clone()));
|
||||
|
||||
let cloudflare_recovery =
|
||||
crate::dispatch::recovery::cloudflare::CloudflareRecovery::new(stores.cookies.clone());
|
||||
let responses = Arc::new(ResponseDispatchService::new(ResponseDispatchServiceParts {
|
||||
account_pool: account_pool.clone(),
|
||||
models: models.clone(),
|
||||
codex: codex.clone(),
|
||||
session_affinity: session_affinity.clone(),
|
||||
recorder,
|
||||
account_identity,
|
||||
cloudflare: cloudflare_recovery,
|
||||
}));
|
||||
let system_update = Arc::new(SystemUpdateService::from_env());
|
||||
let connection_drain = crate::api::middleware::connection_drain::ConnectionDrain::default();
|
||||
|
||||
Ok(Self {
|
||||
database,
|
||||
redis,
|
||||
models,
|
||||
accounts: account_store,
|
||||
client_keys,
|
||||
admin_client_keys,
|
||||
admin_sessions,
|
||||
settings,
|
||||
refresh_policy,
|
||||
admin_accounts,
|
||||
usage_records,
|
||||
ops_errors,
|
||||
usage,
|
||||
account_pool,
|
||||
token_refresh,
|
||||
responses,
|
||||
session_affinity,
|
||||
codex,
|
||||
websocket_pool,
|
||||
fingerprint,
|
||||
account_pseudonymizer,
|
||||
system_update,
|
||||
connection_drain,
|
||||
background_tasks: stores,
|
||||
account_pool_static,
|
||||
})
|
||||
}
|
||||
|
||||
/// 初始化请求热路径依赖的内存状态。
|
||||
pub async fn initialize_hot_path_state(&self) -> Result<(), RuntimeServiceInitializationError> {
|
||||
self.models.reload_from_store().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Services> for crate::api::router::ApiServices {
|
||||
fn from(services: &Services) -> Self {
|
||||
Self {
|
||||
health_probe: Arc::new(crate::bootstrap::state::RuntimeHealthProbe::new(
|
||||
services.database.clone(),
|
||||
services.redis.clone(),
|
||||
)),
|
||||
models: services.models.clone(),
|
||||
accounts: services.accounts.clone(),
|
||||
client_keys: services.client_keys.clone(),
|
||||
admin_client_keys: services.admin_client_keys.clone(),
|
||||
admin_sessions: services.admin_sessions.clone(),
|
||||
settings: services.settings.clone(),
|
||||
admin_accounts: services.admin_accounts.clone(),
|
||||
usage_records: services.usage_records.clone(),
|
||||
ops_errors: services.ops_errors.clone(),
|
||||
usage: services.usage.clone(),
|
||||
account_pool: services.account_pool.clone(),
|
||||
token_refresh: services.token_refresh.clone(),
|
||||
responses: services.responses.clone(),
|
||||
session_affinity: services.session_affinity.clone(),
|
||||
fingerprint: services.fingerprint.clone(),
|
||||
process_control: Arc::new(crate::bootstrap::shutdown::RuntimeProcessControl),
|
||||
system_update: services.system_update.clone(),
|
||||
connection_drain: services.connection_drain.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Services> for crate::api::router::AppState {
|
||||
fn from(services: &Services) -> Self {
|
||||
Self {
|
||||
services: crate::api::router::ApiServices::from(services),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Services> for crate::api::router::AppState {
|
||||
fn from(services: Services) -> Self {
|
||||
Self::from(&services)
|
||||
}
|
||||
}
|
||||
|
||||
fn token_client_config(config: &AppConfig) -> TokenClientConfig {
|
||||
TokenClientConfig {
|
||||
client_id: config.auth.oauth_client_id.clone(),
|
||||
token_endpoint: config.auth.oauth_token_endpoint.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn settings_snapshot_from_config(config: &AppConfig) -> SettingsSnapshot {
|
||||
SettingsSnapshot {
|
||||
model_aliases: config.model_aliases.clone(),
|
||||
refresh_margin_seconds: config.auth.refresh_margin_seconds,
|
||||
refresh_concurrency: config.auth.refresh_concurrency,
|
||||
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
||||
request_interval_ms: config.auth.request_interval_ms,
|
||||
rotation_strategy: config.auth.rotation_strategy.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_settings_to_config(config: &mut AppConfig, settings: &SettingsSnapshot) {
|
||||
config.model_aliases = settings.model_aliases.clone();
|
||||
config.auth.refresh_margin_seconds = settings.refresh_margin_seconds;
|
||||
config.auth.refresh_concurrency = settings.refresh_concurrency;
|
||||
config.auth.max_concurrent_per_account = settings.max_concurrent_per_account;
|
||||
config.auth.request_interval_ms = settings.request_interval_ms;
|
||||
config.auth.rotation_strategy = settings.rotation_strategy.clone();
|
||||
}
|
||||
|
||||
pub fn fingerprint_from_config(
|
||||
config: &crate::bootstrap::config::FingerprintConfig,
|
||||
) -> Fingerprint {
|
||||
Fingerprint {
|
||||
originator: config.originator.clone(),
|
||||
app_version: config.app_version.clone(),
|
||||
build_number: config.build_number.clone(),
|
||||
platform: config.platform.clone(),
|
||||
arch: config.arch.clone(),
|
||||
chromium_version: config.chromium_version.clone(),
|
||||
user_agent_template: config.user_agent_template.clone(),
|
||||
default_headers: config
|
||||
.default_headers
|
||||
.iter()
|
||||
.map(|header| (header.name.clone(), header.value.clone()))
|
||||
.collect(),
|
||||
header_order: config.header_order.clone(),
|
||||
updated_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions {
|
||||
AccountPoolStaticSettings {
|
||||
skip_quota_limited: config.quota.skip_exhausted,
|
||||
tier_priority: config.auth.tier_priority.clone(),
|
||||
}
|
||||
.pool_options(&settings_snapshot_from_config(config))
|
||||
}
|
||||
|
||||
pub async fn run() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
wait_for_scheduled_restart().await;
|
||||
|
||||
let config = AppConfig::load()?;
|
||||
let host = config.server.host.clone();
|
||||
let port = config.server.port;
|
||||
let log_guard = init_logging(&config)?;
|
||||
let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?;
|
||||
let (router, task_coordinator, connection_drain) = build_router(config).await?;
|
||||
let mut task_coordinator = Some(task_coordinator);
|
||||
|
||||
let serve_result = {
|
||||
let (graceful_shutdown_tx, graceful_shutdown_rx) = tokio::sync::oneshot::channel();
|
||||
let serve = axum::serve(
|
||||
listener,
|
||||
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = graceful_shutdown_rx.await;
|
||||
})
|
||||
.into_future();
|
||||
tokio::pin!(serve);
|
||||
tokio::select! {
|
||||
result = &mut serve => result,
|
||||
() = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
let _ = graceful_shutdown_tx.send(());
|
||||
let active_websocket_connections = connection_drain.begin_shutdown();
|
||||
tracing::info!(
|
||||
active_websocket_connections,
|
||||
"正在排空入站连接并关闭运行时任务"
|
||||
);
|
||||
let coordinator = task_coordinator.take();
|
||||
let connection_drain = connection_drain.clone();
|
||||
let runtime_shutdown = async move {
|
||||
let shutdown_tasks = async move {
|
||||
if let Some(coordinator) = coordinator {
|
||||
coordinator.shutdown().await;
|
||||
}
|
||||
};
|
||||
tokio::join!(shutdown_tasks, connection_drain.wait());
|
||||
};
|
||||
let drain_runtime = async {
|
||||
let (serve_result, ()) = tokio::join!(&mut serve, runtime_shutdown);
|
||||
serve_result
|
||||
};
|
||||
tokio::pin!(drain_runtime);
|
||||
tokio::select! {
|
||||
result = &mut drain_runtime => result,
|
||||
() = tokio::time::sleep(HTTP_DRAIN_TIMEOUT) => {
|
||||
tracing::warn!(
|
||||
timeout_secs = HTTP_DRAIN_TIMEOUT.as_secs(),
|
||||
"HTTP graceful shutdown timed out; cancelling remaining connections"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
() = crate::bootstrap::shutdown::shutdown_signal() => {
|
||||
tracing::warn!("received a second shutdown signal; cancelling remaining connections");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(coordinator) = task_coordinator.take() {
|
||||
let active_websocket_connections = connection_drain.begin_shutdown();
|
||||
tracing::info!(
|
||||
active_websocket_connections,
|
||||
"HTTP 服务已退出,正在关闭其余运行时任务"
|
||||
);
|
||||
tokio::join!(coordinator.shutdown(), connection_drain.wait());
|
||||
}
|
||||
serve_result?;
|
||||
|
||||
if let Some(executable_path) = crate::bootstrap::shutdown::restart_executable_path() {
|
||||
tracing::info!(
|
||||
executable = %executable_path.display(),
|
||||
"正在以更新后的二进制替换当前进程"
|
||||
);
|
||||
drop(log_guard);
|
||||
exec_replacement_process(executable_path)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const HTTP_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
|
||||
async fn build_router(
|
||||
mut config: AppConfig,
|
||||
) -> Result<
|
||||
(
|
||||
Router,
|
||||
crate::bootstrap::tasks::coordinator::TaskCoordinator,
|
||||
crate::api::middleware::connection_drain::ConnectionDrain,
|
||||
),
|
||||
Box<dyn Error + Send + Sync>,
|
||||
> {
|
||||
let pool = connect(&config.database.url).await?;
|
||||
let redis = RedisConnection::connect(&config.redis.url, "cpr").await?;
|
||||
let settings =
|
||||
SettingsService::load_or_initialize(settings_snapshot_from_config(&config), &pool).await?;
|
||||
apply_settings_to_config(&mut config, &settings);
|
||||
|
||||
let fingerprint_store = PgFingerprintStore::new(pool.clone());
|
||||
let stores = BackgroundTaskStores {
|
||||
redis: redis.clone(),
|
||||
accounts: PgAccountStore::new(pool.clone()),
|
||||
admin_users: PgAdminUserStore::new(pool.clone()),
|
||||
admin_sessions: RedisAdminSessionStore::new(redis.clone()),
|
||||
cookies: PgCookieStore::new(pool.clone()),
|
||||
fingerprints: fingerprint_store.clone(),
|
||||
session_affinity: RedisSessionAffinityStore::new(redis.clone()),
|
||||
refresh_leases: RedisRefreshLeaseStore::new(redis.clone()),
|
||||
model_snapshots: RedisModelSnapshotStore::new(redis.clone()),
|
||||
client_keys: PgClientKeyStore::new(pool.clone()),
|
||||
usage_records: PgUsageRecordStore::new(pool.clone()),
|
||||
ops_errors: PgOpsErrorLogStore::new(pool.clone()),
|
||||
account_usage: PgAccountUsageStore::new(pool.clone()),
|
||||
request_buckets: PgRequestBucketStore::new(pool),
|
||||
};
|
||||
|
||||
config.admin.validate_default_password()?;
|
||||
let default_admin_password = config.admin.default_password.clone();
|
||||
let data_dir = ensure_data_dir()?;
|
||||
let identity_secret = load_or_create_identity_secret(&data_dir)?;
|
||||
let default_fingerprint = fingerprint_from_config(&config.fingerprint);
|
||||
let runtime_fingerprint = fingerprint_store
|
||||
.ensure_current_seed(&default_fingerprint)
|
||||
.await?;
|
||||
let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint);
|
||||
let runtime_config = crate::bootstrap::state::RuntimeConfig::from(config.clone());
|
||||
let services = Services::try_with_identity_secret_and_usage_record_options(
|
||||
&config,
|
||||
stores,
|
||||
runtime_fingerprint,
|
||||
identity_secret,
|
||||
UsageRecordOptions::from_config(&config),
|
||||
)?;
|
||||
services.initialize_hot_path_state().await?;
|
||||
|
||||
let created_default_admin = services
|
||||
.admin_sessions
|
||||
.ensure_default_admin(&default_admin_password)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
created = created_default_admin,
|
||||
"默认管理员账号已完成初始化检查"
|
||||
);
|
||||
|
||||
let restored_accounts = services.account_pool.restore_from_store().await?;
|
||||
tracing::info!(
|
||||
count = restored_accounts,
|
||||
"运行时账号池已从 PostgreSQL 恢复"
|
||||
);
|
||||
|
||||
let task_coordinator =
|
||||
crate::bootstrap::tasks::coordinator::TaskCoordinator::start(&runtime_config, &services);
|
||||
let app_state = crate::api::AppState::from(&services);
|
||||
let connection_drain = services.connection_drain.clone();
|
||||
let router = crate::api::router::router().with_state(app_state).layer(
|
||||
axum::middleware::from_fn_with_state(
|
||||
connection_drain.clone(),
|
||||
crate::api::middleware::connection_drain::drain_response_body,
|
||||
),
|
||||
);
|
||||
|
||||
Ok((router, task_coordinator, connection_drain))
|
||||
}
|
||||
|
||||
fn init_logging(config: &AppConfig) -> Result<LogGuard, LogError> {
|
||||
const MEBIBYTE: u64 = 1024 * 1024;
|
||||
|
||||
let file = if config.logging.file.enabled {
|
||||
let max_file_size_bytes = config
|
||||
.logging
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(MEBIBYTE)
|
||||
.ok_or(LogError::InvalidConfiguration(
|
||||
"logging.file.max_file_size_mb is too large",
|
||||
))?;
|
||||
Some(RotationConfig::new(
|
||||
&config.logging.file.directory,
|
||||
config.logging.file.retention_days,
|
||||
max_file_size_bytes,
|
||||
config.logging.file.max_files,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
init_tracing(&TracingConfig::new(
|
||||
&config.logging.level,
|
||||
config.logging.stdout,
|
||||
file,
|
||||
))
|
||||
}
|
||||
|
||||
async fn wait_for_scheduled_restart() {
|
||||
let Some(delay_ms) = env::var(RESTART_DELAY_ENV)
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn exec_replacement_process(
|
||||
executable_path: std::path::PathBuf,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::process::Command;
|
||||
|
||||
let error = Command::new(executable_path)
|
||||
.args(env::args_os().skip(1))
|
||||
.exec();
|
||||
Err(Box::new(error))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn exec_replacement_process(
|
||||
_executable_path: std::path::PathBuf,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
Err(Box::new(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"process exec restart is only supported on Unix",
|
||||
)))
|
||||
}
|
||||
@@ -3,18 +3,31 @@
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use tokio::signal;
|
||||
use tokio::sync::broadcast::{self, Receiver};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
static SHUTDOWN_REQUESTS: OnceLock<broadcast::Sender<()>> = OnceLock::new();
|
||||
static RESTART_REQUESTED: AtomicBool = AtomicBool::new(false);
|
||||
static RESTART_EXECUTABLE_PATH: OnceLock<PathBuf> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct RuntimeProcessControl;
|
||||
|
||||
impl crate::api::router::ProcessControl for RuntimeProcessControl {
|
||||
fn request_shutdown(&self) {
|
||||
request_shutdown();
|
||||
}
|
||||
|
||||
fn request_restart(&self, executable_path: PathBuf) {
|
||||
request_process_restart(executable_path);
|
||||
}
|
||||
}
|
||||
|
||||
/// 等待进程关闭信号(Ctrl+C 或 SIGTERM)。
|
||||
pub async fn shutdown_signal() {
|
||||
let mut requested = shutdown_sender().subscribe();
|
||||
@@ -47,11 +60,6 @@ pub fn request_shutdown() {
|
||||
let _ = shutdown_sender().send(());
|
||||
}
|
||||
|
||||
/// 订阅进程关闭请求。
|
||||
pub fn shutdown_subscription() -> Receiver<()> {
|
||||
shutdown_sender().subscribe()
|
||||
}
|
||||
|
||||
/// 请求进程完成优雅关闭后以新二进制替换当前进程。
|
||||
pub fn request_process_restart(executable_path: PathBuf) {
|
||||
let _ = RESTART_EXECUTABLE_PATH.set(executable_path);
|
||||
@@ -0,0 +1,51 @@
|
||||
//! 运行时共享状态。
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{
|
||||
api::router::HealthProbe, bootstrap::config::AppConfig, infra::redis::RedisConnection,
|
||||
};
|
||||
|
||||
/// 运行时配置镜像(从 AppConfig 衍生)。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RuntimeConfig {
|
||||
pub admin: crate::bootstrap::config::AdminConfig,
|
||||
pub auth: crate::bootstrap::config::AuthConfig,
|
||||
pub quota: crate::bootstrap::config::QuotaConfig,
|
||||
}
|
||||
|
||||
/// PostgreSQL 与 Redis 的进程健康检查实现。
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeHealthProbe {
|
||||
database: sqlx::PgPool,
|
||||
redis: RedisConnection,
|
||||
}
|
||||
|
||||
impl RuntimeHealthProbe {
|
||||
pub fn new(database: sqlx::PgPool, redis: RedisConnection) -> Self {
|
||||
Self { database, redis }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HealthProbe for RuntimeHealthProbe {
|
||||
async fn check(&self) -> Result<(), String> {
|
||||
crate::infra::database::ping(&self.database)
|
||||
.await
|
||||
.map_err(|error| format!("PostgreSQL: {error}"))?;
|
||||
self.redis
|
||||
.ping()
|
||||
.await
|
||||
.map_err(|error| format!("Redis: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AppConfig> for RuntimeConfig {
|
||||
fn from(config: AppConfig) -> Self {
|
||||
Self {
|
||||
admin: config.admin.clone(),
|
||||
auth: config.auth.clone(),
|
||||
quota: config.quota,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ use tracing::{debug, info, warn};
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner},
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
|
||||
};
|
||||
|
||||
/// 支持按时间清理过期记录的存储。
|
||||
+7
-7
@@ -4,7 +4,7 @@ use std::future::Future;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::upstream::accounts::cookies::{SqliteCookieStore, SqliteCookieStoreError};
|
||||
use crate::fleet::cookies::{PgCookieStore, PgCookieStoreError};
|
||||
|
||||
use super::{
|
||||
cleanup::{CleanupTask, CleanupTaskMessages, ExpiredCleanupStore},
|
||||
@@ -22,12 +22,12 @@ const MESSAGES: CleanupTaskMessages = CleanupTaskMessages {
|
||||
|
||||
/// Cookie 清理任务。
|
||||
pub struct CookieCleanupTask {
|
||||
task: CleanupTask<SqliteCookieStore>,
|
||||
task: CleanupTask<PgCookieStore>,
|
||||
}
|
||||
|
||||
impl CookieCleanupTask {
|
||||
/// 构造默认 Cookie 清理任务。
|
||||
pub fn new(store: SqliteCookieStore) -> Self {
|
||||
pub fn new(store: PgCookieStore) -> Self {
|
||||
Self {
|
||||
task: CleanupTask::new(store, DEFAULT_INTERVAL_SECS, MESSAGES),
|
||||
}
|
||||
@@ -39,18 +39,18 @@ impl CookieCleanupTask {
|
||||
}
|
||||
|
||||
/// 执行一次清理。
|
||||
pub async fn cleanup_once(&self) -> Result<u64, SqliteCookieStoreError> {
|
||||
pub async fn cleanup_once(&self) -> Result<u64, PgCookieStoreError> {
|
||||
self.task.cleanup_once().await
|
||||
}
|
||||
|
||||
/// 在指定时间点执行一次清理。
|
||||
pub async fn cleanup_once_at(&self, now: DateTime<Utc>) -> Result<u64, SqliteCookieStoreError> {
|
||||
pub async fn cleanup_once_at(&self, now: DateTime<Utc>) -> Result<u64, PgCookieStoreError> {
|
||||
self.task.cleanup_once_at(now).await
|
||||
}
|
||||
}
|
||||
|
||||
impl ExpiredCleanupStore for SqliteCookieStore {
|
||||
type Error = SqliteCookieStoreError;
|
||||
impl ExpiredCleanupStore for PgCookieStore {
|
||||
type Error = PgCookieStoreError;
|
||||
|
||||
fn delete_expired_at(
|
||||
&self,
|
||||
+100
-34
@@ -1,19 +1,18 @@
|
||||
//! 后台任务协调器。
|
||||
|
||||
use std::{path::PathBuf, sync::Arc, time::Duration};
|
||||
use std::{future::Future, path::PathBuf, sync::Arc, time::Duration};
|
||||
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use super::{
|
||||
cookie_cleanup::CookieCleanupTask,
|
||||
fingerprint_update::{
|
||||
start_fingerprint_update_task, CODEX_DESKTOP_APPCAST_URL,
|
||||
DEFAULT_EXTRACTED_FINGERPRINT_PATH,
|
||||
CODEX_DESKTOP_APPCAST_URL, DEFAULT_EXTRACTED_FINGERPRINT_PATH,
|
||||
start_fingerprint_update_task,
|
||||
},
|
||||
model_refresh::ModelRefreshTask,
|
||||
quota_refresh::QuotaRefreshTask,
|
||||
session_affinity_cleanup::SessionAffinityCleanupTask,
|
||||
session_cleanup::SessionCleanupTask,
|
||||
retention_trim::RetentionTrimTask,
|
||||
token_refresh::TokenRefreshTask,
|
||||
};
|
||||
|
||||
@@ -31,10 +30,10 @@ impl TaskCoordinator {
|
||||
|
||||
/// 启动所有后台任务。
|
||||
pub(crate) fn start(
|
||||
config: &crate::runtime::state::RuntimeConfig,
|
||||
services: &crate::runtime::services::Services,
|
||||
config: &crate::bootstrap::state::RuntimeConfig,
|
||||
services: &crate::bootstrap::services::Services,
|
||||
) -> Self {
|
||||
let mut coordinator = TaskCoordinator::default();
|
||||
let mut coordinator = Self::start_settings_subscriptions(services);
|
||||
let stores = &services.background_tasks;
|
||||
|
||||
coordinator.push(
|
||||
@@ -42,26 +41,34 @@ impl TaskCoordinator {
|
||||
CookieCleanupTask::new(stores.cookies.clone()).start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"session_cleanup",
|
||||
SessionCleanupTask::new(
|
||||
stores.admin_sessions.clone(),
|
||||
config.admin.session_cleanup_interval_secs,
|
||||
)
|
||||
.start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"session_affinity_cleanup",
|
||||
SessionAffinityCleanupTask::new(
|
||||
stores.session_affinity.clone(),
|
||||
config.admin.session_cleanup_interval_secs,
|
||||
"retention_trim",
|
||||
RetentionTrimTask::new(
|
||||
stores.usage_records.clone(),
|
||||
stores.ops_errors.clone(),
|
||||
stores.request_buckets.clone(),
|
||||
stores.fingerprints.clone(),
|
||||
)
|
||||
.start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"model_refresh",
|
||||
ModelRefreshTask::new(services.models.clone(), services.account_pool.clone())
|
||||
.with_installation_id(services.installation_id.clone())
|
||||
.start(),
|
||||
ModelRefreshTask::new(
|
||||
services.models.clone(),
|
||||
services.account_pool.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
)
|
||||
.start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"model_etag_refresh",
|
||||
SchedulerHandle::spawn(
|
||||
ModelRefreshTask::new(
|
||||
services.models.clone(),
|
||||
services.account_pool.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
)
|
||||
.run_etag_refreshes(),
|
||||
),
|
||||
);
|
||||
if config.auth.refresh_enabled {
|
||||
coordinator.push(
|
||||
@@ -73,7 +80,9 @@ impl TaskCoordinator {
|
||||
"quota_refresh",
|
||||
QuotaRefreshTask::with_intervals(
|
||||
stores.accounts.clone(),
|
||||
stores.account_usage.clone(),
|
||||
services.codex.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
config
|
||||
.quota
|
||||
.refresh_interval_minutes
|
||||
@@ -81,21 +90,21 @@ impl TaskCoordinator {
|
||||
.max(1),
|
||||
30 * 60,
|
||||
)
|
||||
.with_installation_id(services.installation_id.clone())
|
||||
.with_cookie_store(stores.cookies.clone())
|
||||
.with_account_pool(services.account_pool.clone())
|
||||
.start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"fingerprint_update",
|
||||
coordinator.push("fingerprint_update", {
|
||||
let fingerprint = services.fingerprint.snapshot();
|
||||
start_fingerprint_update_task(
|
||||
Some(stores.fingerprints.clone()),
|
||||
stores.fingerprints.clone(),
|
||||
services.fingerprint.clone(),
|
||||
CODEX_DESKTOP_APPCAST_URL.to_string(),
|
||||
PathBuf::from(DEFAULT_EXTRACTED_FINGERPRINT_PATH),
|
||||
services.fingerprint.app_version.clone(),
|
||||
services.fingerprint.build_number.clone(),
|
||||
),
|
||||
);
|
||||
fingerprint.app_version,
|
||||
fingerprint.build_number,
|
||||
)
|
||||
});
|
||||
if let Some(pool) = &services.websocket_pool {
|
||||
coordinator.push(
|
||||
"websocket_pool",
|
||||
@@ -106,12 +115,55 @@ impl TaskCoordinator {
|
||||
coordinator
|
||||
}
|
||||
|
||||
/// 启动各领域的设置订阅任务。
|
||||
pub fn start_settings_subscriptions(services: &crate::bootstrap::services::Services) -> Self {
|
||||
let mut coordinator = TaskCoordinator::default();
|
||||
coordinator.push(
|
||||
"model_settings",
|
||||
SchedulerHandle::spawn(
|
||||
services
|
||||
.models
|
||||
.clone()
|
||||
.subscribe_settings(services.settings.subscribe()),
|
||||
),
|
||||
);
|
||||
coordinator.push(
|
||||
"account_pool_settings",
|
||||
SchedulerHandle::spawn(services.account_pool.clone().subscribe_settings(
|
||||
services.settings.subscribe(),
|
||||
services.account_pool_static.clone(),
|
||||
)),
|
||||
);
|
||||
coordinator.push(
|
||||
"refresh_policy_settings",
|
||||
SchedulerHandle::spawn(
|
||||
services
|
||||
.refresh_policy
|
||||
.clone()
|
||||
.subscribe_settings(services.settings.subscribe()),
|
||||
),
|
||||
);
|
||||
coordinator
|
||||
}
|
||||
|
||||
/// 关闭所有后台任务。
|
||||
pub async fn shutdown(self) {
|
||||
tracing::info!("正在关闭后台任务");
|
||||
for (name, handle) in self.handles {
|
||||
let shutdown_tasks = self.handles.into_iter().map(|(name, handle)| async move {
|
||||
handle.shutdown().await;
|
||||
tracing::info!(task = name, "后台任务已停止");
|
||||
});
|
||||
if tokio::time::timeout(
|
||||
COORDINATOR_SHUTDOWN_TIMEOUT,
|
||||
futures::future::join_all(shutdown_tasks),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
timeout_secs = COORDINATOR_SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"后台任务并行关闭超时"
|
||||
);
|
||||
}
|
||||
tracing::info!("所有后台任务已停止");
|
||||
}
|
||||
@@ -125,10 +177,21 @@ pub enum SchedulerHandle {
|
||||
handle: JoinHandle<()>,
|
||||
},
|
||||
/// 关闭上游 WebSocket 连接池。
|
||||
WebSocketPool(Arc<crate::upstream::transport::CodexWebSocketPool>),
|
||||
WebSocketPool(Arc<crate::upstream::openai::transport::CodexWebSocketPool>),
|
||||
}
|
||||
|
||||
impl SchedulerHandle {
|
||||
fn spawn(future: impl Future<Output = ()> + Send + 'static) -> Self {
|
||||
let (shutdown_tx, mut shutdown_rx) = tokio::sync::mpsc::channel(1);
|
||||
let handle = tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
() = future => {}
|
||||
_ = shutdown_rx.recv() => {}
|
||||
}
|
||||
});
|
||||
Self::new(shutdown_tx, handle)
|
||||
}
|
||||
|
||||
/// 使用关闭 channel 构造句柄。
|
||||
pub(crate) fn new(shutdown_tx: tokio::sync::mpsc::Sender<()>, handle: JoinHandle<()>) -> Self {
|
||||
Self::Channel {
|
||||
@@ -138,7 +201,9 @@ impl SchedulerHandle {
|
||||
}
|
||||
|
||||
/// 使用 WebSocket 连接池构造句柄。
|
||||
pub fn from_websocket_pool(pool: Arc<crate::upstream::transport::CodexWebSocketPool>) -> Self {
|
||||
pub fn from_websocket_pool(
|
||||
pool: Arc<crate::upstream::openai::transport::CodexWebSocketPool>,
|
||||
) -> Self {
|
||||
Self::WebSocketPool(pool)
|
||||
}
|
||||
|
||||
@@ -160,6 +225,7 @@ impl SchedulerHandle {
|
||||
}
|
||||
|
||||
const TASK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const COORDINATOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(6);
|
||||
|
||||
async fn wait_for_task_shutdown(handle: JoinHandle<()>) {
|
||||
let mut handle = handle;
|
||||
+28
-10
@@ -2,13 +2,15 @@
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tracing::warn;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::upstream::fingerprint::{FingerprintRepository, UpdateChecker, APPCAST_POLL_INTERVAL};
|
||||
use crate::upstream::openai::fingerprint::{
|
||||
APPCAST_POLL_INTERVAL, PgFingerprintStore, RuntimeFingerprint, UpdateChecker,
|
||||
};
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner},
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
|
||||
};
|
||||
|
||||
/// Codex Desktop 官方 appcast 地址。
|
||||
@@ -19,14 +21,15 @@ pub const DEFAULT_EXTRACTED_FINGERPRINT_PATH: &str = "data/extracted-fingerprint
|
||||
|
||||
/// 启动指纹自动更新后台任务。
|
||||
pub fn start_fingerprint_update_task(
|
||||
repository: Option<FingerprintRepository>,
|
||||
store: PgFingerprintStore,
|
||||
runtime_fingerprint: RuntimeFingerprint,
|
||||
appcast_url: String,
|
||||
extracted_fingerprint_path: PathBuf,
|
||||
current_version: String,
|
||||
current_build: String,
|
||||
) -> SchedulerHandle {
|
||||
let update_checker = UpdateChecker::with_client(
|
||||
repository,
|
||||
store,
|
||||
reqwest::Client::new(),
|
||||
appcast_url,
|
||||
extracted_fingerprint_path,
|
||||
@@ -36,6 +39,7 @@ pub fn start_fingerprint_update_task(
|
||||
spawn_periodic_task(
|
||||
FingerprintUpdateTask {
|
||||
checker: update_checker,
|
||||
runtime_fingerprint,
|
||||
first_tick: true,
|
||||
},
|
||||
PeriodicTaskConfig::new(
|
||||
@@ -48,6 +52,7 @@ pub fn start_fingerprint_update_task(
|
||||
|
||||
struct FingerprintUpdateTask {
|
||||
checker: UpdateChecker,
|
||||
runtime_fingerprint: RuntimeFingerprint,
|
||||
first_tick: bool,
|
||||
}
|
||||
|
||||
@@ -55,11 +60,24 @@ impl PeriodicTaskRunner for FingerprintUpdateTask {
|
||||
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
let first_tick = std::mem::take(&mut self.first_tick);
|
||||
if let Err(error) = self.checker.check_and_apply_update().await {
|
||||
if first_tick {
|
||||
warn!(error = %error, "fingerprint 首次版本检查失败");
|
||||
} else {
|
||||
warn!(error = %error, "fingerprint 定期版本检查失败");
|
||||
match self.checker.check_and_apply_update().await {
|
||||
Ok(Some(fingerprint)) => {
|
||||
let app_version = fingerprint.app_version.clone();
|
||||
let build_number = fingerprint.build_number.clone();
|
||||
self.runtime_fingerprint.replace(fingerprint);
|
||||
info!(
|
||||
app_version = %app_version,
|
||||
build_number = %build_number,
|
||||
"fingerprint 运行时版本已更新"
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
if first_tick {
|
||||
warn!(error = %error, "fingerprint 首次版本检查失败");
|
||||
} else {
|
||||
warn!(error = %error, "fingerprint 定期版本检查失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -7,6 +7,5 @@ pub mod fingerprint_update;
|
||||
pub mod model_refresh;
|
||||
mod periodic;
|
||||
pub mod quota_refresh;
|
||||
pub mod session_affinity_cleanup;
|
||||
pub mod session_cleanup;
|
||||
pub mod retention_trim;
|
||||
pub mod token_refresh;
|
||||
+50
-25
@@ -7,25 +7,26 @@ use tokio::sync::mpsc;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::upstream::accounts::pool::RuntimeAccountPoolService;
|
||||
use crate::upstream::models::service::{
|
||||
use crate::fleet::pool::AccountPoolService;
|
||||
use crate::infra::identity::AccountPseudonymizer;
|
||||
use crate::models::service::{
|
||||
ModelRefreshPlanAccount, ModelRefreshResult, ModelService, ModelServiceError,
|
||||
};
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner, PeriodicTaskStartup},
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, PeriodicTaskStartup, spawn_periodic_task},
|
||||
};
|
||||
|
||||
/// 模型刷新任务接线器。
|
||||
pub struct ModelRefreshTask {
|
||||
model_service: Arc<ModelService>,
|
||||
account_pool: Arc<RuntimeAccountPoolService>,
|
||||
account_pool: Arc<AccountPoolService>,
|
||||
refresh_interval_secs: u64,
|
||||
initial_delay_ms: u64,
|
||||
retry_delay_ms: u64,
|
||||
max_retries: u32,
|
||||
installation_id: Option<String>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
}
|
||||
|
||||
const DEFAULT_REFRESH_INTERVAL_SECS: u64 = 3600;
|
||||
@@ -37,7 +38,8 @@ impl ModelRefreshTask {
|
||||
/// 构造默认任务。
|
||||
pub fn new(
|
||||
model_service: Arc<ModelService>,
|
||||
account_pool: Arc<RuntimeAccountPoolService>,
|
||||
account_pool: Arc<AccountPoolService>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
model_service,
|
||||
@@ -46,16 +48,10 @@ impl ModelRefreshTask {
|
||||
initial_delay_ms: INITIAL_DELAY_MS,
|
||||
retry_delay_ms: RETRY_DELAY_MS,
|
||||
max_retries: MAX_RETRIES,
|
||||
installation_id: None,
|
||||
account_pseudonymizer,
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置 Codex installation id。
|
||||
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
|
||||
self.installation_id = installation_id.filter(|id| !id.trim().is_empty());
|
||||
self
|
||||
}
|
||||
|
||||
/// 启动后台刷新任务。
|
||||
pub fn start(self) -> SchedulerHandle {
|
||||
let config = PeriodicTaskConfig::new(
|
||||
@@ -67,6 +63,27 @@ impl ModelRefreshTask {
|
||||
spawn_periodic_task(self, config)
|
||||
}
|
||||
|
||||
/// 在 Responses 响应声明模型目录 ETag 变化时异步刷新。
|
||||
pub async fn run_etag_refreshes(self) {
|
||||
let mut etag_changes = self.model_service.subscribe_models_etag_changes();
|
||||
while etag_changes.changed().await.is_ok() {
|
||||
// 合并同一批并发响应产生的重复通知。
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
etag_changes.borrow_and_update();
|
||||
match self.refresh_once().await {
|
||||
Ok(result) => info!(
|
||||
refreshed_plans = result.refreshed_plans,
|
||||
model_count = result.model_count,
|
||||
"模型 ETag 变化刷新完成"
|
||||
),
|
||||
Err(ModelServiceError::NoAccounts) => {
|
||||
info!("模型 ETag 变化刷新跳过:没有可用账号");
|
||||
}
|
||||
Err(error) => warn!(error = ?error, "模型 ETag 变化刷新失败"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一次模型刷新并同步账号池模型计划约束。
|
||||
pub async fn refresh_once(&self) -> Result<ModelRefreshResult, ModelServiceError> {
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
@@ -75,20 +92,20 @@ impl ModelRefreshTask {
|
||||
.iter()
|
||||
.map(|selected| ModelRefreshPlanAccount {
|
||||
plan_type: selected.plan_type.clone(),
|
||||
account: selected.account.clone(),
|
||||
access_token: selected.account.access_token.clone(),
|
||||
account_id: selected.account.account_id.clone(),
|
||||
installation_id: self
|
||||
.account_pseudonymizer
|
||||
.installation_id(&selected.account.id),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let result = self
|
||||
.model_service
|
||||
.refresh_backend_models_with_installation_id(
|
||||
&plan_accounts,
|
||||
&request_id,
|
||||
self.installation_id.as_deref(),
|
||||
)
|
||||
.refresh_backend_models(&plan_accounts, &request_id)
|
||||
.await;
|
||||
for selected in &selected_accounts {
|
||||
self.account_pool.release(&selected.account.id).await;
|
||||
for selected in selected_accounts {
|
||||
selected.release().await;
|
||||
}
|
||||
|
||||
let result = result?;
|
||||
@@ -116,6 +133,7 @@ impl PeriodicTaskRunner for ModelRefreshTask {
|
||||
|
||||
let mut attempt = 0;
|
||||
let mut has_fetched_once = false;
|
||||
let mut deferred_until_accounts_available = false;
|
||||
|
||||
while attempt < self.max_retries {
|
||||
match self.refresh_once().await {
|
||||
@@ -131,7 +149,8 @@ impl PeriodicTaskRunner for ModelRefreshTask {
|
||||
Err(error) => {
|
||||
attempt += 1;
|
||||
if matches!(error, ModelServiceError::NoAccounts) {
|
||||
warn!("首次模型刷新跳过:没有可用账户,将进入周期重试");
|
||||
info!("首次模型刷新跳过:没有可用账户,将进入周期重试");
|
||||
deferred_until_accounts_available = true;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -155,7 +174,7 @@ impl PeriodicTaskRunner for ModelRefreshTask {
|
||||
}
|
||||
}
|
||||
|
||||
if !has_fetched_once {
|
||||
if !has_fetched_once && !deferred_until_accounts_available {
|
||||
warn!("模型刷新任务首次尝试全部失败,切换到周期刷新");
|
||||
}
|
||||
|
||||
@@ -165,8 +184,14 @@ impl PeriodicTaskRunner for ModelRefreshTask {
|
||||
|
||||
fn tick(&mut self) -> super::periodic::TaskFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
if let Err(error) = self.refresh_once().await {
|
||||
warn!(error = ?error, "刷新模型列表失败");
|
||||
match self.refresh_once().await {
|
||||
Ok(_) => {}
|
||||
Err(ModelServiceError::NoAccounts) => {
|
||||
info!("模型刷新跳过:没有可用账户");
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = ?error, "刷新模型列表失败");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
+19
-18
@@ -6,23 +6,25 @@ use tokio::time::Instant;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::{
|
||||
upstream::accounts::{
|
||||
cookies::SqliteCookieStore,
|
||||
pool::RuntimeAccountPoolService,
|
||||
quota::{QuotaRefreshSummary, RuntimeQuotaRefreshService},
|
||||
store::SqliteAccountStore,
|
||||
fleet::{
|
||||
cookies::PgCookieStore,
|
||||
pool::AccountPoolService,
|
||||
quota::{QuotaRefreshService, QuotaRefreshSummary},
|
||||
store::PgAccountStore,
|
||||
},
|
||||
upstream::transport::CodexBackendClient,
|
||||
infra::identity::AccountPseudonymizer,
|
||||
telemetry::account_usage::store::PgAccountUsageStore,
|
||||
upstream::openai::transport::CodexBackendClient,
|
||||
};
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner},
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
|
||||
};
|
||||
|
||||
/// 主动配额刷新后台任务。
|
||||
pub struct QuotaRefreshTask {
|
||||
service: RuntimeQuotaRefreshService,
|
||||
service: QuotaRefreshService,
|
||||
interval_secs: u64,
|
||||
last_refreshed: HashMap<String, Instant>,
|
||||
}
|
||||
@@ -30,15 +32,19 @@ pub struct QuotaRefreshTask {
|
||||
impl QuotaRefreshTask {
|
||||
/// 使用自定义间隔构造配额刷新后台任务。
|
||||
pub fn with_intervals(
|
||||
store: SqliteAccountStore,
|
||||
store: PgAccountStore,
|
||||
usage_store: PgAccountUsageStore,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
interval_secs: u64,
|
||||
min_refresh_interval_secs: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
service: RuntimeQuotaRefreshService::with_min_refresh_interval_secs(
|
||||
service: QuotaRefreshService::with_min_refresh_interval_secs(
|
||||
store,
|
||||
usage_store,
|
||||
codex,
|
||||
account_pseudonymizer,
|
||||
min_refresh_interval_secs,
|
||||
),
|
||||
interval_secs,
|
||||
@@ -46,20 +52,14 @@ impl QuotaRefreshTask {
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置 Codex installation id。
|
||||
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
|
||||
self.service = self.service.with_installation_id(installation_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 usage 请求可复用的账号 Cookie 存储。
|
||||
pub fn with_cookie_store(mut self, cookie_store: SqliteCookieStore) -> Self {
|
||||
pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self {
|
||||
self.service = self.service.with_cookie_store(cookie_store);
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置运行时账号池,用于刷新后同步内存调度状态。
|
||||
pub fn with_account_pool(mut self, account_pool: Arc<RuntimeAccountPoolService>) -> Self {
|
||||
pub fn with_account_pool(mut self, account_pool: Arc<AccountPoolService>) -> Self {
|
||||
self.service = self.service.with_account_pool(account_pool);
|
||||
self
|
||||
}
|
||||
@@ -89,6 +89,7 @@ impl PeriodicTaskRunner for QuotaRefreshTask {
|
||||
Ok(summary) => {
|
||||
debug!(
|
||||
candidates = summary.candidates,
|
||||
skipped_cooldown = summary.skipped_cooldown,
|
||||
skipped_recent = summary.skipped_recent,
|
||||
failed = summary.failed,
|
||||
"没有需要刷新的 quota 锁定或待验证账号"
|
||||
@@ -0,0 +1,91 @@
|
||||
//! PostgreSQL 增长表保留期清理任务。
|
||||
|
||||
use chrono::Utc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
telemetry::{
|
||||
buckets::store::PgRequestBucketStore, ops::store::PgOpsErrorLogStore,
|
||||
usage::store::PgUsageRecordStore,
|
||||
},
|
||||
upstream::openai::fingerprint::PgFingerprintStore,
|
||||
};
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, TaskFuture, spawn_periodic_task},
|
||||
};
|
||||
|
||||
const RETENTION_TRIM_INTERVAL_SECS: u64 = 60 * 60;
|
||||
const FINGERPRINT_HISTORY_LIMIT: std::num::NonZeroU32 = std::num::NonZeroU32::new(100).unwrap();
|
||||
|
||||
pub struct RetentionTrimTask {
|
||||
usage_records: PgUsageRecordStore,
|
||||
ops_errors: PgOpsErrorLogStore,
|
||||
buckets: PgRequestBucketStore,
|
||||
fingerprints: PgFingerprintStore,
|
||||
}
|
||||
|
||||
impl RetentionTrimTask {
|
||||
pub fn new(
|
||||
usage_records: PgUsageRecordStore,
|
||||
ops_errors: PgOpsErrorLogStore,
|
||||
buckets: PgRequestBucketStore,
|
||||
fingerprints: PgFingerprintStore,
|
||||
) -> Self {
|
||||
Self {
|
||||
usage_records,
|
||||
ops_errors,
|
||||
buckets,
|
||||
fingerprints,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(self) -> SchedulerHandle {
|
||||
spawn_periodic_task(
|
||||
self,
|
||||
PeriodicTaskConfig::new(
|
||||
RETENTION_TRIM_INTERVAL_SECS,
|
||||
"PostgreSQL 保留期清理任务已启动",
|
||||
"PostgreSQL 保留期清理任务已关闭",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// 立即执行一轮全部 PostgreSQL 保留期清理。
|
||||
pub async fn run_once(&self) {
|
||||
let now = Utc::now();
|
||||
match self.usage_records.trim_to_retention(now).await {
|
||||
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期成功使用事实"),
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(error = %error, "清理成功使用事实失败"),
|
||||
}
|
||||
match self.ops_errors.trim_to_retention(now).await {
|
||||
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期运维错误事实"),
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(error = %error, "清理运维错误事实失败"),
|
||||
}
|
||||
match self.buckets.trim_to_retention(now).await {
|
||||
Ok(deleted) if deleted > 0 => info!(deleted, "已清理过期请求时间桶"),
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(error = %error, "清理请求时间桶失败"),
|
||||
}
|
||||
match self
|
||||
.fingerprints
|
||||
.trim_update_history_to_limit(FINGERPRINT_HISTORY_LIMIT)
|
||||
.await
|
||||
{
|
||||
Ok(deleted) if deleted > 0 => {
|
||||
info!(deleted, "已清理超限指纹更新历史")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(error = %error, "清理超限指纹更新历史失败"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PeriodicTaskRunner for RetentionTrimTask {
|
||||
fn tick(&mut self) -> TaskFuture<'_, ()> {
|
||||
Box::pin(self.run_once())
|
||||
}
|
||||
}
|
||||
+11
-13
@@ -2,16 +2,15 @@
|
||||
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::upstream::accounts::{
|
||||
store::SqliteAccountStore,
|
||||
token_refresh::{
|
||||
RefreshLeaseStore, RuntimeRefreshPolicy, RuntimeTokenRefreshService, TokenRefresher,
|
||||
},
|
||||
use crate::fleet::{
|
||||
refresh::{RedisRefreshLeaseStore, RuntimeRefreshPolicy, TokenRefreshService},
|
||||
store::PgAccountStore,
|
||||
};
|
||||
use crate::upstream::openai::token_client::TokenRefresher;
|
||||
|
||||
use super::{
|
||||
coordinator::SchedulerHandle,
|
||||
periodic::{spawn_periodic_task, PeriodicTaskConfig, PeriodicTaskRunner},
|
||||
periodic::{PeriodicTaskConfig, PeriodicTaskRunner, spawn_periodic_task},
|
||||
};
|
||||
|
||||
const DEFAULT_INTERVAL_SECS: u64 = 60;
|
||||
@@ -21,7 +20,7 @@ pub struct TokenRefreshTask<C>
|
||||
where
|
||||
C: TokenRefresher,
|
||||
{
|
||||
service: RuntimeTokenRefreshService<C>,
|
||||
service: TokenRefreshService<C>,
|
||||
interval_secs: u64,
|
||||
}
|
||||
|
||||
@@ -30,15 +29,15 @@ where
|
||||
C: TokenRefresher,
|
||||
{
|
||||
/// 构造默认后台任务。
|
||||
pub fn new(store: SqliteAccountStore, policy: RuntimeRefreshPolicy, client: C) -> Self {
|
||||
pub fn new(store: PgAccountStore, policy: RuntimeRefreshPolicy, client: C) -> Self {
|
||||
Self {
|
||||
service: RuntimeTokenRefreshService::new(store, policy, client),
|
||||
service: TokenRefreshService::new(store, policy, client),
|
||||
interval_secs: DEFAULT_INTERVAL_SECS,
|
||||
}
|
||||
}
|
||||
|
||||
/// 使用已创建的运行时刷新服务构造后台任务。
|
||||
pub fn from_service(service: RuntimeTokenRefreshService<C>) -> Self {
|
||||
pub fn from_service(service: TokenRefreshService<C>) -> Self {
|
||||
Self {
|
||||
service,
|
||||
interval_secs: DEFAULT_INTERVAL_SECS,
|
||||
@@ -46,7 +45,7 @@ where
|
||||
}
|
||||
|
||||
/// 使用刷新租约存储保护账号刷新。
|
||||
pub fn with_refresh_lease_store(mut self, refresh_leases: RefreshLeaseStore) -> Self {
|
||||
pub fn with_refresh_lease_store(mut self, refresh_leases: RedisRefreshLeaseStore) -> Self {
|
||||
self.service = self.service.with_refresh_lease_store(refresh_leases);
|
||||
self
|
||||
}
|
||||
@@ -73,7 +72,6 @@ where
|
||||
info!(
|
||||
scheduled = summary.scheduled,
|
||||
immediate = summary.immediate,
|
||||
recovery_scheduled = summary.recovery_scheduled,
|
||||
replaced = summary.replaced,
|
||||
"token 刷新定时器已调度"
|
||||
);
|
||||
@@ -94,7 +92,7 @@ where
|
||||
|
||||
fn shutdown(&mut self) -> super::periodic::TaskFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.service.clear_scheduled_timers().await;
|
||||
self.service.shutdown().await;
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
use std::{env, path::Path};
|
||||
|
||||
use super::schema::AppConfig;
|
||||
|
||||
const CONFIG_FILE_ENV: &str = "CPR_CONFIG_FILE";
|
||||
|
||||
impl AppConfig {
|
||||
/// 从 `CPR_CONFIG_FILE` 或当前目录 `config.yaml` 加载配置文件。
|
||||
pub fn load() -> Result<Self, ::config::ConfigError> {
|
||||
let config_file = env::var(CONFIG_FILE_ENV)
|
||||
.ok()
|
||||
.filter(|value| !value.trim().is_empty());
|
||||
match config_file {
|
||||
Some(path) => load_file(path),
|
||||
None => Self::load_from_dir("."),
|
||||
}
|
||||
}
|
||||
|
||||
/// 从指定目录加载配置文件。
|
||||
pub fn load_from_dir(config_dir: impl AsRef<Path>) -> Result<Self, ::config::ConfigError> {
|
||||
let config_dir = config_dir.as_ref();
|
||||
load_file(config_dir.join("config.yaml"))
|
||||
}
|
||||
}
|
||||
|
||||
fn load_file(config_file: impl AsRef<Path>) -> Result<AppConfig, ::config::ConfigError> {
|
||||
::config::Config::builder()
|
||||
.add_source(::config::File::from(config_file.as_ref()).required(true))
|
||||
.build()?
|
||||
.try_deserialize()
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
//! 配置加载与配置 schema。
|
||||
|
||||
#![deny(missing_docs)]
|
||||
|
||||
/// 配置加载器。
|
||||
pub mod loader;
|
||||
/// 配置文件 schema 与默认值。
|
||||
pub mod schema;
|
||||
/// 管理端可变设置与运行时写回服务。
|
||||
pub mod settings;
|
||||
@@ -1,704 +0,0 @@
|
||||
//! 配置设置领域逻辑与运行时数据库服务。
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, RwLock as StdRwLock},
|
||||
};
|
||||
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{sqlite::SqliteRow, Row, SqlitePool};
|
||||
use subtle::ConstantTimeEq;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::config::schema::AppConfig;
|
||||
use crate::infra::identity::generate_admin_api_key;
|
||||
use crate::upstream::accounts::{
|
||||
pool::{AccountPoolOptions, RotationStrategy, RuntimeAccountPoolService},
|
||||
token_refresh::{RefreshPolicy, RuntimeRefreshPolicy},
|
||||
};
|
||||
|
||||
const ROTATION_STRATEGIES: [&str; 3] = ["least_used", "round_robin", "sticky"];
|
||||
const RUNTIME_SETTINGS_ID: i64 = 1;
|
||||
|
||||
/// 管理端可变设置。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AdminSettings {
|
||||
/// 模型别名映射。
|
||||
pub model_aliases: BTreeMap<String, String>,
|
||||
/// 模型到账号 ID 的显式路由。
|
||||
pub model_account_routes: BTreeMap<String, Vec<String>>,
|
||||
/// 访问令牌过期前多少秒开始刷新。
|
||||
pub refresh_margin_seconds: u64,
|
||||
/// 访问令牌刷新并发数。
|
||||
pub refresh_concurrency: u32,
|
||||
/// 单账号最大并发请求数。
|
||||
pub max_concurrent_per_account: usize,
|
||||
/// 同账号请求间隔毫秒数。
|
||||
pub request_interval_ms: u64,
|
||||
/// 账号轮换策略。
|
||||
pub rotation_strategy: String,
|
||||
}
|
||||
|
||||
impl Default for AdminSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
model_aliases: BTreeMap::new(),
|
||||
model_account_routes: BTreeMap::new(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
rotation_strategy: "least_used".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端设置补丁。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdminSettingsPatch {
|
||||
/// 模型别名映射。
|
||||
pub model_aliases: Option<BTreeMap<String, String>>,
|
||||
/// 模型到账号 ID 的显式路由。
|
||||
pub model_account_routes: Option<BTreeMap<String, Vec<String>>>,
|
||||
/// 访问令牌过期前多少秒开始刷新。
|
||||
pub refresh_margin_seconds: Option<u64>,
|
||||
/// 访问令牌刷新并发数。
|
||||
pub refresh_concurrency: Option<u32>,
|
||||
/// 单账号最大并发请求数。
|
||||
pub max_concurrent_per_account: Option<usize>,
|
||||
/// 同账号请求间隔毫秒数。
|
||||
pub request_interval_ms: Option<u64>,
|
||||
/// 账号轮换策略。
|
||||
pub rotation_strategy: Option<String>,
|
||||
}
|
||||
|
||||
/// 设置领域服务。
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SettingsService;
|
||||
|
||||
impl SettingsService {
|
||||
/// 将管理端设置补丁应用到当前设置。
|
||||
pub fn apply_patch(
|
||||
current: &mut AdminSettings,
|
||||
patch: AdminSettingsPatch,
|
||||
) -> Result<(), SettingsServiceError> {
|
||||
if let Some(model_aliases) = patch.model_aliases {
|
||||
current.model_aliases = validate_model_aliases(model_aliases)?;
|
||||
}
|
||||
if let Some(model_account_routes) = patch.model_account_routes {
|
||||
current.model_account_routes = validate_model_account_routes(model_account_routes)?;
|
||||
}
|
||||
if let Some(refresh_margin_seconds) = patch.refresh_margin_seconds {
|
||||
current.refresh_margin_seconds =
|
||||
positive_u64("refreshMarginSeconds", refresh_margin_seconds)?;
|
||||
}
|
||||
if let Some(refresh_concurrency) = patch.refresh_concurrency {
|
||||
current.refresh_concurrency = positive_u32("refreshConcurrency", refresh_concurrency)?;
|
||||
}
|
||||
if let Some(max_concurrent_per_account) = patch.max_concurrent_per_account {
|
||||
current.max_concurrent_per_account =
|
||||
positive_usize("maxConcurrentPerAccount", max_concurrent_per_account)?;
|
||||
}
|
||||
if let Some(request_interval_ms) = patch.request_interval_ms {
|
||||
current.request_interval_ms = request_interval_ms;
|
||||
}
|
||||
if let Some(rotation_strategy) = patch.rotation_strategy {
|
||||
current.rotation_strategy = validate_rotation_strategy(&rotation_strategy)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置领域错误。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum SettingsServiceError {
|
||||
/// 字段值无效。
|
||||
#[error("invalid setting `{field}`: {message}")]
|
||||
InvalidField {
|
||||
/// 字段名。
|
||||
field: String,
|
||||
/// 错误说明。
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl SettingsServiceError {
|
||||
/// 返回无效字段名。
|
||||
pub fn field(&self) -> &str {
|
||||
match self {
|
||||
Self::InvalidField { field, .. } => field,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_model_aliases(
|
||||
aliases: BTreeMap<String, String>,
|
||||
) -> Result<BTreeMap<String, String>, SettingsServiceError> {
|
||||
let mut normalized = BTreeMap::new();
|
||||
for (alias, target) in aliases {
|
||||
let alias = non_empty_string("modelAliases", &alias)?;
|
||||
let target = non_empty_string("modelAliases", &target)?;
|
||||
if alias == target {
|
||||
return Err(invalid_field(
|
||||
"modelAliases",
|
||||
"alias and target must differ",
|
||||
));
|
||||
}
|
||||
normalized.insert(alias, target);
|
||||
}
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
fn validate_model_account_routes(
|
||||
routes: BTreeMap<String, Vec<String>>,
|
||||
) -> Result<BTreeMap<String, Vec<String>>, SettingsServiceError> {
|
||||
let mut normalized = BTreeMap::new();
|
||||
for (model, account_ids) in routes {
|
||||
let model = non_empty_string("modelAccountRoutes", &model)?;
|
||||
let mut normalized_account_ids = Vec::new();
|
||||
for account_id in account_ids {
|
||||
let account_id = non_empty_string("modelAccountRoutes", &account_id)?;
|
||||
if normalized_account_ids.contains(&account_id) {
|
||||
return Err(invalid_field(
|
||||
"modelAccountRoutes",
|
||||
format!("duplicate account id `{account_id}`"),
|
||||
));
|
||||
}
|
||||
normalized_account_ids.push(account_id);
|
||||
}
|
||||
if normalized_account_ids.is_empty() {
|
||||
return Err(invalid_field(
|
||||
"modelAccountRoutes",
|
||||
"account list must not be empty",
|
||||
));
|
||||
}
|
||||
normalized.insert(model, normalized_account_ids);
|
||||
}
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
fn validate_rotation_strategy(strategy: &str) -> Result<String, SettingsServiceError> {
|
||||
let strategy = non_empty_string("rotationStrategy", strategy)?;
|
||||
if ROTATION_STRATEGIES.contains(&strategy.as_str()) {
|
||||
Ok(strategy)
|
||||
} else {
|
||||
Err(invalid_field(
|
||||
"rotationStrategy",
|
||||
"must be one of least_used, round_robin, sticky",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty_string(field: &str, value: &str) -> Result<String, SettingsServiceError> {
|
||||
let value = value.trim().to_string();
|
||||
if value.is_empty() {
|
||||
Err(invalid_field(field, "must not be empty"))
|
||||
} else {
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_u64(field: &str, value: u64) -> Result<u64, SettingsServiceError> {
|
||||
if value == 0 {
|
||||
Err(invalid_field(field, "must be greater than 0"))
|
||||
} else {
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_u32(field: &str, value: u32) -> Result<u32, SettingsServiceError> {
|
||||
if value == 0 {
|
||||
Err(invalid_field(field, "must be greater than 0"))
|
||||
} else {
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_usize(field: &str, value: usize) -> Result<usize, SettingsServiceError> {
|
||||
if value == 0 {
|
||||
Err(invalid_field(field, "must be greater than 0"))
|
||||
} else {
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_field(field: &str, message: impl Into<String>) -> SettingsServiceError {
|
||||
SettingsServiceError::InvalidField {
|
||||
field: field.to_string(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理员 API Key 状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdminApiKeyStatus {
|
||||
/// 是否已经生成。
|
||||
pub exists: bool,
|
||||
/// 脱敏后的密钥。
|
||||
pub masked_key: Option<String>,
|
||||
}
|
||||
|
||||
/// 运行时设置服务。
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeSettingsService {
|
||||
current: Arc<StdRwLock<Arc<AppConfig>>>,
|
||||
pool: SqlitePool,
|
||||
account_pool: Arc<RuntimeAccountPoolService>,
|
||||
refresh_policy: RuntimeRefreshPolicy,
|
||||
}
|
||||
|
||||
impl RuntimeSettingsService {
|
||||
/// 构造运行时设置服务。
|
||||
pub fn new(
|
||||
config: AppConfig,
|
||||
pool: SqlitePool,
|
||||
account_pool: Arc<RuntimeAccountPoolService>,
|
||||
refresh_policy: RuntimeRefreshPolicy,
|
||||
) -> Self {
|
||||
Self {
|
||||
current: Arc::new(StdRwLock::new(Arc::new(config))),
|
||||
pool,
|
||||
account_pool,
|
||||
refresh_policy,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回当前运行时配置快照。
|
||||
pub fn current(&self) -> Arc<AppConfig> {
|
||||
self.current
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// 初始化缺省运行设置,并返回数据库中的运行配置快照。
|
||||
pub async fn load_or_initialize_config(
|
||||
mut config: AppConfig,
|
||||
pool: &SqlitePool,
|
||||
) -> Result<AppConfig, RuntimeSettingsError> {
|
||||
let settings = admin_settings_from_config(&config);
|
||||
insert_runtime_settings_if_missing(pool, &settings).await?;
|
||||
let settings = load_runtime_settings(pool).await?;
|
||||
apply_admin_settings_to_config(&mut config, settings);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// 应用设置补丁、写入数据库并更新运行时配置快照。
|
||||
pub async fn update(
|
||||
&self,
|
||||
patch: AdminSettingsPatch,
|
||||
) -> Result<Arc<AppConfig>, RuntimeSettingsError> {
|
||||
let mut next = (*self.current()).clone();
|
||||
let mut settings = admin_settings_from_config(&next);
|
||||
SettingsService::apply_patch(&mut settings, patch)?;
|
||||
save_runtime_settings(&self.pool, &settings).await?;
|
||||
apply_admin_settings_to_config(&mut next, settings);
|
||||
self.account_pool
|
||||
.apply_options(
|
||||
account_pool_options_from_config(&next),
|
||||
next.auth.request_interval_ms,
|
||||
)
|
||||
.await;
|
||||
self.refresh_policy.update(RefreshPolicy {
|
||||
refresh_margin_seconds: next.auth.refresh_margin_seconds,
|
||||
refresh_concurrency: next.auth.refresh_concurrency,
|
||||
});
|
||||
let next = Arc::new(next);
|
||||
*self
|
||||
.current
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = next.clone();
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// 返回管理员 API Key 状态。
|
||||
pub async fn admin_api_key_status(&self) -> Result<AdminApiKeyStatus, RuntimeSettingsError> {
|
||||
admin_api_key_status(&self.pool).await
|
||||
}
|
||||
|
||||
/// 重新生成管理员 API Key,完整值只在本次返回。
|
||||
pub async fn regenerate_admin_api_key(&self) -> Result<String, RuntimeSettingsError> {
|
||||
let settings = admin_settings_from_config(&self.current());
|
||||
insert_runtime_settings_if_missing(&self.pool, &settings).await?;
|
||||
|
||||
let key = generate_admin_api_key();
|
||||
sqlx::query(
|
||||
r"
|
||||
update runtime_settings
|
||||
set admin_api_key = ?, updated_at = ?
|
||||
where id = ?",
|
||||
)
|
||||
.bind(&key)
|
||||
.bind(Utc::now().to_rfc3339())
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
/// 删除管理员 API Key。
|
||||
pub async fn delete_admin_api_key(&self) -> Result<(), RuntimeSettingsError> {
|
||||
sqlx::query(
|
||||
r"
|
||||
update runtime_settings
|
||||
set admin_api_key = null, updated_at = ?
|
||||
where id = ?",
|
||||
)
|
||||
.bind(Utc::now().to_rfc3339())
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验管理员 API Key。
|
||||
pub async fn verify_admin_api_key(&self, key: &str) -> Result<bool, RuntimeSettingsError> {
|
||||
if key.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let stored = load_admin_api_key(&self.pool).await?;
|
||||
Ok(stored
|
||||
.as_deref()
|
||||
.filter(|stored_key| !stored_key.is_empty())
|
||||
.is_some_and(|stored_key| key.as_bytes().ct_eq(stored_key.as_bytes()).into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// 运行时设置错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum RuntimeSettingsError {
|
||||
/// 设置字段校验失败。
|
||||
#[error(transparent)]
|
||||
InvalidField(#[from] SettingsServiceError),
|
||||
/// 数据库操作失败。
|
||||
#[error("runtime settings database error: {0}")]
|
||||
Database(#[from] sqlx::Error),
|
||||
/// JSON 编解码失败。
|
||||
#[error("runtime settings json error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
/// 数据库存储值非法。
|
||||
#[error("invalid stored setting `{field}`: {message}")]
|
||||
StoredField {
|
||||
/// 字段名。
|
||||
field: String,
|
||||
/// 错误说明。
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
fn admin_settings_from_config(config: &AppConfig) -> AdminSettings {
|
||||
AdminSettings {
|
||||
model_aliases: config.model_aliases.clone(),
|
||||
model_account_routes: config.model_account_routes.clone(),
|
||||
refresh_margin_seconds: config.auth.refresh_margin_seconds,
|
||||
refresh_concurrency: config.auth.refresh_concurrency,
|
||||
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
||||
request_interval_ms: config.auth.request_interval_ms,
|
||||
rotation_strategy: config.auth.rotation_strategy.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_admin_settings_to_config(config: &mut AppConfig, settings: AdminSettings) {
|
||||
config.model_aliases = settings.model_aliases;
|
||||
config.model_account_routes = settings.model_account_routes;
|
||||
config.auth.refresh_margin_seconds = settings.refresh_margin_seconds;
|
||||
config.auth.refresh_concurrency = settings.refresh_concurrency;
|
||||
config.auth.max_concurrent_per_account = settings.max_concurrent_per_account;
|
||||
config.auth.request_interval_ms = settings.request_interval_ms;
|
||||
config.auth.rotation_strategy = settings.rotation_strategy;
|
||||
}
|
||||
|
||||
async fn insert_runtime_settings_if_missing(
|
||||
pool: &SqlitePool,
|
||||
settings: &AdminSettings,
|
||||
) -> Result<(), RuntimeSettingsError> {
|
||||
let model_aliases_json = serde_json::to_string(&settings.model_aliases)?;
|
||||
let mut tx = pool.begin().await?;
|
||||
let result = sqlx::query(
|
||||
r"
|
||||
insert or ignore into runtime_settings (
|
||||
id,
|
||||
model_aliases_json,
|
||||
refresh_margin_seconds,
|
||||
refresh_concurrency,
|
||||
max_concurrent_per_account,
|
||||
request_interval_ms,
|
||||
rotation_strategy,
|
||||
updated_at
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.bind(model_aliases_json)
|
||||
.bind(
|
||||
i64::try_from(settings.refresh_margin_seconds)
|
||||
.map_err(|_| stored_field_error("refreshMarginSeconds", "out of range"))?,
|
||||
)
|
||||
.bind(i64::from(settings.refresh_concurrency))
|
||||
.bind(
|
||||
i64::try_from(settings.max_concurrent_per_account)
|
||||
.map_err(|_| stored_field_error("maxConcurrentPerAccount", "out of range"))?,
|
||||
)
|
||||
.bind(
|
||||
i64::try_from(settings.request_interval_ms)
|
||||
.map_err(|_| stored_field_error("requestIntervalMs", "out of range"))?,
|
||||
)
|
||||
.bind(&settings.rotation_strategy)
|
||||
.bind(Utc::now().to_rfc3339())
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
if result.rows_affected() > 0 {
|
||||
replace_model_account_routes(&mut tx, &settings.model_account_routes).await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_runtime_settings(
|
||||
pool: &SqlitePool,
|
||||
settings: &AdminSettings,
|
||||
) -> Result<(), RuntimeSettingsError> {
|
||||
let model_aliases_json = serde_json::to_string(&settings.model_aliases)?;
|
||||
let mut tx = pool.begin().await?;
|
||||
sqlx::query(
|
||||
r"
|
||||
insert into runtime_settings (
|
||||
id,
|
||||
model_aliases_json,
|
||||
refresh_margin_seconds,
|
||||
refresh_concurrency,
|
||||
max_concurrent_per_account,
|
||||
request_interval_ms,
|
||||
rotation_strategy,
|
||||
updated_at
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
on conflict(id) do update set
|
||||
model_aliases_json = excluded.model_aliases_json,
|
||||
refresh_margin_seconds = excluded.refresh_margin_seconds,
|
||||
refresh_concurrency = excluded.refresh_concurrency,
|
||||
max_concurrent_per_account = excluded.max_concurrent_per_account,
|
||||
request_interval_ms = excluded.request_interval_ms,
|
||||
rotation_strategy = excluded.rotation_strategy,
|
||||
updated_at = excluded.updated_at",
|
||||
)
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.bind(model_aliases_json)
|
||||
.bind(
|
||||
i64::try_from(settings.refresh_margin_seconds)
|
||||
.map_err(|_| stored_field_error("refreshMarginSeconds", "out of range"))?,
|
||||
)
|
||||
.bind(i64::from(settings.refresh_concurrency))
|
||||
.bind(
|
||||
i64::try_from(settings.max_concurrent_per_account)
|
||||
.map_err(|_| stored_field_error("maxConcurrentPerAccount", "out of range"))?,
|
||||
)
|
||||
.bind(
|
||||
i64::try_from(settings.request_interval_ms)
|
||||
.map_err(|_| stored_field_error("requestIntervalMs", "out of range"))?,
|
||||
)
|
||||
.bind(&settings.rotation_strategy)
|
||||
.bind(Utc::now().to_rfc3339())
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
replace_model_account_routes(&mut tx, &settings.model_account_routes).await?;
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_runtime_settings(pool: &SqlitePool) -> Result<AdminSettings, RuntimeSettingsError> {
|
||||
let row = sqlx::query(
|
||||
r"
|
||||
select
|
||||
model_aliases_json,
|
||||
refresh_margin_seconds,
|
||||
refresh_concurrency,
|
||||
max_concurrent_per_account,
|
||||
request_interval_ms,
|
||||
rotation_strategy
|
||||
from runtime_settings
|
||||
where id = ?",
|
||||
)
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let mut settings = runtime_settings_from_row(&row)?;
|
||||
settings.model_account_routes = load_model_account_routes(pool).await?;
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
async fn admin_api_key_status(
|
||||
pool: &SqlitePool,
|
||||
) -> Result<AdminApiKeyStatus, RuntimeSettingsError> {
|
||||
let key = load_admin_api_key(pool).await?;
|
||||
let masked_key = key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.map(mask_api_key);
|
||||
Ok(AdminApiKeyStatus {
|
||||
exists: masked_key.is_some(),
|
||||
masked_key,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_admin_api_key(pool: &SqlitePool) -> Result<Option<String>, RuntimeSettingsError> {
|
||||
Ok(sqlx::query_scalar::<_, Option<String>>(
|
||||
"select admin_api_key from runtime_settings where id = ?",
|
||||
)
|
||||
.bind(RUNTIME_SETTINGS_ID)
|
||||
.fetch_optional(pool)
|
||||
.await?
|
||||
.flatten())
|
||||
}
|
||||
|
||||
fn mask_api_key(key: &str) -> String {
|
||||
if key.len() > 14 {
|
||||
format!("{}...{}", &key[..10], &key[key.len() - 4..])
|
||||
} else {
|
||||
key.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime_settings_from_row(row: &SqliteRow) -> Result<AdminSettings, RuntimeSettingsError> {
|
||||
let model_aliases_json: String = row.get("model_aliases_json");
|
||||
Ok(AdminSettings {
|
||||
model_aliases: serde_json::from_str(&model_aliases_json)?,
|
||||
model_account_routes: BTreeMap::new(),
|
||||
refresh_margin_seconds: positive_i64_to_u64(
|
||||
"refreshMarginSeconds",
|
||||
row.get("refresh_margin_seconds"),
|
||||
)?,
|
||||
refresh_concurrency: positive_i64_to_u32(
|
||||
"refreshConcurrency",
|
||||
row.get("refresh_concurrency"),
|
||||
)?,
|
||||
max_concurrent_per_account: positive_i64_to_usize(
|
||||
"maxConcurrentPerAccount",
|
||||
row.get("max_concurrent_per_account"),
|
||||
)?,
|
||||
request_interval_ms: non_negative_i64_to_u64(
|
||||
"requestIntervalMs",
|
||||
row.get("request_interval_ms"),
|
||||
)?,
|
||||
rotation_strategy: validate_rotation_strategy(&row.get::<String, _>("rotation_strategy"))?,
|
||||
})
|
||||
}
|
||||
|
||||
async fn replace_model_account_routes(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
routes: &BTreeMap<String, Vec<String>>,
|
||||
) -> Result<(), RuntimeSettingsError> {
|
||||
sqlx::query("delete from model_account_routes")
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
let now = Utc::now().to_rfc3339();
|
||||
for (model, account_ids) in routes {
|
||||
for (priority, account_id) in account_ids.iter().enumerate() {
|
||||
sqlx::query(
|
||||
r"
|
||||
insert into model_account_routes (
|
||||
model,
|
||||
account_id,
|
||||
priority,
|
||||
enabled,
|
||||
created_at,
|
||||
updated_at
|
||||
) values (?, ?, ?, 1, ?, ?)",
|
||||
)
|
||||
.bind(model)
|
||||
.bind(account_id)
|
||||
.bind(
|
||||
i64::try_from(priority)
|
||||
.map_err(|_| stored_field_error("modelAccountRoutes", "out of range"))?,
|
||||
)
|
||||
.bind(&now)
|
||||
.bind(&now)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_model_account_routes(
|
||||
pool: &SqlitePool,
|
||||
) -> Result<BTreeMap<String, Vec<String>>, RuntimeSettingsError> {
|
||||
let rows: Vec<(String, String)> = sqlx::query_as(
|
||||
r"
|
||||
select model, account_id
|
||||
from model_account_routes
|
||||
where enabled = 1
|
||||
order by model asc, priority asc, account_id asc",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
let mut routes = BTreeMap::<String, Vec<String>>::new();
|
||||
for (model, account_id) in rows {
|
||||
routes.entry(model).or_default().push(account_id);
|
||||
}
|
||||
Ok(routes)
|
||||
}
|
||||
|
||||
/// 从当前运行配置生成账号池调度参数。
|
||||
pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions {
|
||||
AccountPoolOptions {
|
||||
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
||||
rotation_strategy: rotation_strategy_from_config(&config.auth.rotation_strategy),
|
||||
skip_quota_limited: config.quota.skip_exhausted,
|
||||
tier_priority: config.auth.tier_priority.clone(),
|
||||
model_account_routes: config.model_account_routes.clone(),
|
||||
..AccountPoolOptions::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn rotation_strategy_from_config(strategy: &str) -> RotationStrategy {
|
||||
match strategy {
|
||||
"round_robin" => RotationStrategy::RoundRobin,
|
||||
"sticky" => RotationStrategy::Sticky,
|
||||
_ => RotationStrategy::LeastUsed,
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_i64_to_u64(field: &'static str, value: i64) -> Result<u64, RuntimeSettingsError> {
|
||||
if value <= 0 {
|
||||
return Err(stored_field_error(field, "must be greater than 0"));
|
||||
}
|
||||
u64::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
|
||||
}
|
||||
|
||||
fn positive_i64_to_u32(field: &'static str, value: i64) -> Result<u32, RuntimeSettingsError> {
|
||||
if value <= 0 {
|
||||
return Err(stored_field_error(field, "must be greater than 0"));
|
||||
}
|
||||
u32::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
|
||||
}
|
||||
|
||||
fn positive_i64_to_usize(field: &'static str, value: i64) -> Result<usize, RuntimeSettingsError> {
|
||||
if value <= 0 {
|
||||
return Err(stored_field_error(field, "must be greater than 0"));
|
||||
}
|
||||
usize::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
|
||||
}
|
||||
|
||||
fn non_negative_i64_to_u64(field: &'static str, value: i64) -> Result<u64, RuntimeSettingsError> {
|
||||
if value < 0 {
|
||||
return Err(stored_field_error(
|
||||
field,
|
||||
"must be greater than or equal to 0",
|
||||
));
|
||||
}
|
||||
u64::try_from(value).map_err(|_| stored_field_error(field, "out of range"))
|
||||
}
|
||||
|
||||
fn stored_field_error(
|
||||
field: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) -> RuntimeSettingsError {
|
||||
RuntimeSettingsError::StoredField {
|
||||
field: field.into(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//! 客户端身份、本地会话索引与账号作用域 wire 身份隔离。
|
||||
|
||||
use crate::{
|
||||
infra::identity::AccountPseudonymizer,
|
||||
upstream::openai::protocol::responses::CodexResponsesRequest,
|
||||
};
|
||||
|
||||
/// 持久化密钥驱动的账号身份隔离服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AccountIdentityService {
|
||||
pseudonymizer: AccountPseudonymizer,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AccountIdentityService {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountIdentityService")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// 单次账号 attempt 的 wire 身份。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountScopedIdentity {
|
||||
pub prompt_cache_key: Option<String>,
|
||||
pub session_id: Option<String>,
|
||||
pub thread_id: Option<String>,
|
||||
pub client_request_id: String,
|
||||
pub turn_id: Option<String>,
|
||||
pub window_id: Option<String>,
|
||||
pub parent_thread_id: Option<String>,
|
||||
pub installation_id: String,
|
||||
}
|
||||
|
||||
struct IdentityValues<'a> {
|
||||
prompt_cache_key: Option<&'a str>,
|
||||
session_id: Option<&'a str>,
|
||||
thread_id: Option<&'a str>,
|
||||
client_request_id: Option<&'a str>,
|
||||
turn_id: Option<&'a str>,
|
||||
window_id: Option<&'a str>,
|
||||
parent_thread_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl AccountIdentityService {
|
||||
pub fn new(pseudonymizer: AccountPseudonymizer) -> Self {
|
||||
Self { pseudonymizer }
|
||||
}
|
||||
|
||||
pub fn prepare_local_identity(&self, request: &mut CodexResponsesRequest) {
|
||||
if request.local_conversation_id.is_some() {
|
||||
return;
|
||||
}
|
||||
let anchor = request
|
||||
.prompt_cache_key()
|
||||
.or(request.client_session_id.as_deref())
|
||||
.or(request.client_thread_id.as_deref())
|
||||
.or(request.client_conversation_id.as_deref())
|
||||
.map(ToString::to_string)
|
||||
.or_else(|| super::resolve::derive_stable_conversation_key(request));
|
||||
request.local_conversation_id = anchor
|
||||
.as_deref()
|
||||
.map(|anchor| format!("lc_{}", self.pseudonym("local-conversation", None, anchor)));
|
||||
}
|
||||
|
||||
pub fn scope(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
self.scope_values(
|
||||
account_id,
|
||||
proxy_request_id,
|
||||
IdentityValues {
|
||||
prompt_cache_key: request.prompt_cache_key(),
|
||||
session_id: request.client_session_id.as_deref(),
|
||||
thread_id: request.client_thread_id.as_deref(),
|
||||
client_request_id: request.client_request_id.as_deref(),
|
||||
turn_id: request.client_turn_id.as_deref(),
|
||||
window_id: request.codex_window_id.as_deref(),
|
||||
parent_thread_id: request.parent_thread_id.as_deref(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn scope_auxiliary(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym("client-request-id", Some(account_id), proxy_request_id)
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
..AccountScopedIdentity::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_values(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
values: IdentityValues<'_>,
|
||||
) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
prompt_cache_key: self.scope_optional(
|
||||
"prompt-cache-key",
|
||||
account_id,
|
||||
values.prompt_cache_key,
|
||||
),
|
||||
session_id: self.scope_optional("session-id", account_id, values.session_id),
|
||||
thread_id: self.scope_optional("thread-id", account_id, values.thread_id),
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym(
|
||||
"client-request-id",
|
||||
Some(account_id),
|
||||
values.client_request_id.unwrap_or(proxy_request_id),
|
||||
)
|
||||
),
|
||||
turn_id: self.scope_optional("turn-id", account_id, values.turn_id),
|
||||
window_id: self.scope_optional("window-id", account_id, values.window_id),
|
||||
parent_thread_id: self.scope_optional(
|
||||
"parent-thread-id",
|
||||
account_id,
|
||||
values.parent_thread_id,
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_optional(
|
||||
&self,
|
||||
domain: &str,
|
||||
account_id: &str,
|
||||
value: Option<&str>,
|
||||
) -> Option<String> {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| format!("wi_{}", self.pseudonym(domain, Some(account_id), value)))
|
||||
}
|
||||
|
||||
fn pseudonym(&self, domain: &str, account_id: Option<&str>, value: &str) -> String {
|
||||
self.pseudonymizer.scoped(domain, account_id, value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//! 会话亲和身份、存储、解析与运行时服务。
|
||||
|
||||
pub mod identity;
|
||||
pub mod resolve;
|
||||
pub mod service;
|
||||
pub mod store;
|
||||
pub mod types;
|
||||
|
||||
pub use identity::{AccountIdentityService, AccountScopedIdentity};
|
||||
pub use resolve::*;
|
||||
pub use service::*;
|
||||
pub use store::*;
|
||||
pub use types::*;
|
||||
@@ -0,0 +1,246 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::{SessionAffinityEntry, SessionAffinityService},
|
||||
recovery::history::HistoryRecoveryPlan,
|
||||
service::ResponseDispatchService,
|
||||
},
|
||||
upstream::openai::protocol::{
|
||||
events::TokenUsage,
|
||||
responses::{CodexResponsesRequest, completed_response_metadata},
|
||||
},
|
||||
};
|
||||
|
||||
impl ResponseDispatchService {
|
||||
pub(in crate::dispatch) async fn preferred_account_id_for_request(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
history: &HistoryRecoveryPlan,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
if let Some(account_id) = history.preferred_account_id() {
|
||||
return Some(account_id.to_string());
|
||||
}
|
||||
if request.previous_response_id().is_some() {
|
||||
return None;
|
||||
}
|
||||
self.conversation_account_id(request, now).await
|
||||
}
|
||||
|
||||
async fn conversation_account_id(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
let conversation_id = request
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let variant_hash = compute_variant_hash(request);
|
||||
self.session_affinity
|
||||
.lookup_latest_account_by_conversation(conversation_id, None, Some(&variant_hash), now)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn record_response_affinity(
|
||||
session_affinity: &SessionAffinityService,
|
||||
history: &HistoryRecoveryPlan,
|
||||
original_request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
body: &str,
|
||||
turn_state: Option<String>,
|
||||
usage: Option<TokenUsage>,
|
||||
) {
|
||||
let metadata = match completed_response_metadata(body) {
|
||||
Ok(Some(metadata)) => metadata,
|
||||
Ok(None) => return,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
"failed to parse completed response metadata for session affinity"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let conversation_id = history
|
||||
.conversation_id(original_request)
|
||||
.unwrap_or(&metadata.response_id)
|
||||
.to_string();
|
||||
let variant_hash = compute_variant_hash(original_request);
|
||||
let entry = SessionAffinityEntry {
|
||||
account_id: account_id.to_string(),
|
||||
conversation_id,
|
||||
turn_state: turn_state
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| original_request.turn_state.clone()),
|
||||
instructions_hash: Some(hash_instructions(Some(original_request.instructions()))),
|
||||
input_tokens: usage.map(|usage| usage.input_tokens),
|
||||
function_call_ids: metadata.function_call_ids,
|
||||
variant_hash: Some(variant_hash),
|
||||
continuation_scope: if original_request.store() {
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::Persisted
|
||||
} else {
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal
|
||||
},
|
||||
created_at: Utc::now(),
|
||||
};
|
||||
if let Err(error) = session_affinity
|
||||
.record(metadata.response_id.clone(), entry)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
response_id = %metadata.response_id,
|
||||
account_id = %account_id,
|
||||
"failed to record session affinity"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Variant identity helpers
|
||||
// ====================================================================
|
||||
|
||||
/// 准备用于区分并发分支的变体身份。
|
||||
pub fn prepare_variant_identity(request: &mut CodexResponsesRequest) {
|
||||
request.variant_identity = build_variant_identity(request);
|
||||
}
|
||||
|
||||
/// 计算请求变体哈希。
|
||||
pub fn compute_variant_hash(request: &CodexResponsesRequest) -> String {
|
||||
compute_variant_hash_with_identity(request, request.variant_identity.as_deref())
|
||||
}
|
||||
|
||||
fn compute_variant_hash_with_identity(
|
||||
request: &CodexResponsesRequest,
|
||||
identity: Option<&str>,
|
||||
) -> String {
|
||||
let tools_json = request.tools().map_or_else(|| "[]".to_string(), tools_json);
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(request.instructions().as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(tools_json.as_bytes());
|
||||
if let Some(identity) = identity
|
||||
.map(str::trim)
|
||||
.filter(|identity| !identity.is_empty())
|
||||
{
|
||||
hasher.update(b"\0");
|
||||
hasher.update(identity.as_bytes());
|
||||
}
|
||||
hex::encode(hasher.finalize()).chars().take(12).collect()
|
||||
}
|
||||
|
||||
fn tools_json(tools: &[Value]) -> String {
|
||||
serde_json::to_string(tools).unwrap_or_else(|_| "[]".to_string())
|
||||
}
|
||||
|
||||
fn build_variant_identity(request: &CodexResponsesRequest) -> Option<String> {
|
||||
let mut parts = Vec::with_capacity(2);
|
||||
if let Some(window_id) = non_empty_str(request.codex_window_id.as_deref()) {
|
||||
parts.push(format!("window:{window_id}"));
|
||||
}
|
||||
if (request.explicit_prompt_cache_key
|
||||
|| non_empty_str(request.client_conversation_id.as_deref()).is_some())
|
||||
&& let Some(anchor) = derive_stable_conversation_key(request)
|
||||
{
|
||||
parts.push(format!("anchor:{anchor}"));
|
||||
}
|
||||
|
||||
(!parts.is_empty()).then(|| parts.join("\0"))
|
||||
}
|
||||
|
||||
fn non_empty_str(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
/// 计算 instructions 哈希。
|
||||
pub fn hash_instructions(instructions: Option<&str>) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(instructions.unwrap_or_default().as_bytes());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
const LEADING_SYSTEM_REMINDER_OPEN: &str = "<system-reminder>";
|
||||
const LEADING_SYSTEM_REMINDER_CLOSE: &str = "</system-reminder>";
|
||||
|
||||
/// 按原版 `stable-conversation-key.ts` 的规则派生稳定 conversation key。
|
||||
pub fn derive_stable_conversation_key(request: &CodexResponsesRequest) -> Option<String> {
|
||||
let instructions = request
|
||||
.instructions()
|
||||
.chars()
|
||||
.take(2000)
|
||||
.collect::<String>();
|
||||
let first_user_text = first_user_text(request.input());
|
||||
let normalized_first_user_text = normalize_conversation_anchor_text(&first_user_text);
|
||||
let first_user_text = if normalized_first_user_text.is_empty() {
|
||||
first_user_text
|
||||
} else {
|
||||
normalized_first_user_text
|
||||
};
|
||||
if instructions.is_empty() && first_user_text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(request.model().as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(instructions.as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(first_user_text.as_bytes());
|
||||
let hex = hex::encode(hasher.finalize());
|
||||
|
||||
Some(format!(
|
||||
"{}-{}-{}-{}-{}",
|
||||
&hex[0..8],
|
||||
&hex[8..12],
|
||||
&hex[12..16],
|
||||
&hex[16..20],
|
||||
&hex[20..32]
|
||||
))
|
||||
}
|
||||
|
||||
fn first_user_text(input: &[Value]) -> String {
|
||||
for item in input {
|
||||
if item.get("role").and_then(Value::as_str) != Some("user") {
|
||||
continue;
|
||||
}
|
||||
let Some(content) = item.get("content") else {
|
||||
return String::new();
|
||||
};
|
||||
if let Some(text) = content.as_str() {
|
||||
return text.to_string();
|
||||
}
|
||||
if let Some(parts) = content.as_array() {
|
||||
return parts
|
||||
.iter()
|
||||
.filter(|part| part.get("type").and_then(Value::as_str) == Some("input_text"))
|
||||
.filter_map(|part| part.get("text").and_then(Value::as_str))
|
||||
.collect::<String>();
|
||||
}
|
||||
return String::new();
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn normalize_conversation_anchor_text(text: &str) -> String {
|
||||
let mut rest = text.trim_start();
|
||||
loop {
|
||||
let lower = rest.to_ascii_lowercase();
|
||||
if !lower.starts_with(LEADING_SYSTEM_REMINDER_OPEN) {
|
||||
break;
|
||||
}
|
||||
let Some(close_start) = lower.find(LEADING_SYSTEM_REMINDER_CLOSE) else {
|
||||
break;
|
||||
};
|
||||
let close_end = close_start + LEADING_SYSTEM_REMINDER_CLOSE.len();
|
||||
rest = rest[close_end..].trim_start();
|
||||
}
|
||||
rest.to_string()
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//! 运行时会话亲和服务。
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::{
|
||||
store::{RedisSessionAffinityStore, RedisSessionAffinityStoreError},
|
||||
types::SessionAffinityEntry,
|
||||
};
|
||||
|
||||
const DEFAULT_SESSION_AFFINITY_TTL_SECS: i64 = 4 * 60 * 60;
|
||||
|
||||
/// 运行时会话亲和性服务。
|
||||
#[derive(Clone)]
|
||||
pub struct SessionAffinityService {
|
||||
store: RedisSessionAffinityStore,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
/// 运行时会话亲和性错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SessionAffinityError {
|
||||
#[error("session affinity store error: {0}")]
|
||||
Store(#[from] RedisSessionAffinityStoreError),
|
||||
}
|
||||
|
||||
impl SessionAffinityService {
|
||||
pub fn new(store: RedisSessionAffinityStore) -> Self {
|
||||
let ttl = Duration::seconds(DEFAULT_SESSION_AFFINITY_TTL_SECS);
|
||||
Self { store, ttl }
|
||||
}
|
||||
|
||||
pub async fn record(
|
||||
&self,
|
||||
response_id: String,
|
||||
entry: SessionAffinityEntry,
|
||||
) -> Result<(), SessionAffinityError> {
|
||||
Ok(self.store.upsert(&response_id, &entry, self.ttl).await?)
|
||||
}
|
||||
|
||||
pub async fn lookup(
|
||||
&self,
|
||||
response_id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<SessionAffinityEntry> {
|
||||
self.entry(response_id, now).await
|
||||
}
|
||||
|
||||
pub async fn lookup_latest_account_by_conversation(
|
||||
&self,
|
||||
conversation_id: &str,
|
||||
max_age: Option<Duration>,
|
||||
variant_hash: Option<&str>,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
self.latest(conversation_id, max_age, variant_hash, now)
|
||||
.await
|
||||
.map(|(_, entry)| entry.account_id)
|
||||
}
|
||||
|
||||
pub async fn forget(&self, response_id: &str) -> bool {
|
||||
match self.store.forget(response_id).await {
|
||||
Ok(forgotten) => forgotten,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
response_id,
|
||||
error = %error,
|
||||
"failed to remove Redis session affinity"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn forget_account(&self, account_id: &str) -> bool {
|
||||
match self.store.forget_account(account_id).await {
|
||||
Ok(forgotten) => forgotten > 0,
|
||||
Err(error) => {
|
||||
tracing::warn!(account_id, error = %error, "failed to remove account affinities");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn entry(&self, response_id: &str, now: DateTime<Utc>) -> Option<SessionAffinityEntry> {
|
||||
match self.store.get(response_id, now, self.ttl).await {
|
||||
Ok(entry) => entry,
|
||||
Err(error) => {
|
||||
tracing::warn!(response_id, error = %error, "failed to read Redis session affinity");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn latest(
|
||||
&self,
|
||||
conversation_id: &str,
|
||||
max_age: Option<Duration>,
|
||||
variant_hash: Option<&str>,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<(String, SessionAffinityEntry)> {
|
||||
match self
|
||||
.store
|
||||
.latest_by_conversation(conversation_id, max_age, variant_hash, now, self.ttl)
|
||||
.await
|
||||
{
|
||||
Ok(entry) => entry,
|
||||
Err(error) => {
|
||||
tracing::warn!(conversation_id, error = %error, "failed to query Redis affinity index");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,499 @@
|
||||
//! Redis 会话亲和存储。
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use redis::AsyncCommands;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::infra::redis::RedisConnection;
|
||||
|
||||
use super::types::SessionAffinityEntry;
|
||||
|
||||
/// 单会话保留的最近响应元数据数量。
|
||||
pub const MAX_CONVERSATION_AFFINITIES: usize = 128;
|
||||
/// 单账号保留的最近响应索引数量。
|
||||
pub const MAX_ACCOUNT_AFFINITIES: usize = 16_384;
|
||||
/// Redis 全局保留的最近响应元数据数量。
|
||||
pub const MAX_GLOBAL_AFFINITIES: usize = 65_536;
|
||||
/// Redis 全局响应元数据 JSON 字节预算。
|
||||
pub const MAX_GLOBAL_AFFINITY_BYTES: u64 = 128 * 1024 * 1024;
|
||||
|
||||
const MAX_AFFINITY_METADATA_BYTES: usize = 64 * 1024;
|
||||
const GLOBAL_STALE_CLEANUP_BATCH: usize = 256;
|
||||
const AFFINITY_VERSION: &str = "affinity:v3";
|
||||
|
||||
const UPSERT_SCRIPT: &str = r#"
|
||||
local response_id = ARGV[1]
|
||||
local metadata_json = ARGV[2]
|
||||
local score = tonumber(ARGV[3])
|
||||
local expires_at = tonumber(ARGV[4])
|
||||
local ttl_seconds = tonumber(ARGV[5])
|
||||
local cutoff_ms = tonumber(ARGV[6])
|
||||
local response_prefix = ARGV[7]
|
||||
local conversation_prefix = ARGV[8]
|
||||
local account_prefix = ARGV[9]
|
||||
local max_conversation = tonumber(ARGV[10])
|
||||
local max_account = tonumber(ARGV[11])
|
||||
local max_global = tonumber(ARGV[12])
|
||||
local max_global_bytes = tonumber(ARGV[13])
|
||||
local stale_cleanup_batch = tonumber(ARGV[14])
|
||||
|
||||
local current = cjson.decode(metadata_json)
|
||||
local conversation_id = current.conversationId
|
||||
local account_id = current.accountId
|
||||
|
||||
local function decrement_bytes(size)
|
||||
if size <= 0 then
|
||||
return
|
||||
end
|
||||
local remaining = redis.call('DECRBY', KEYS[5], size)
|
||||
if remaining <= 0 then
|
||||
redis.call('DEL', KEYS[5])
|
||||
end
|
||||
end
|
||||
|
||||
local function read_info(id)
|
||||
local raw = redis.call('HGET', KEYS[6], id)
|
||||
if not raw then
|
||||
return nil
|
||||
end
|
||||
local ok, info = pcall(cjson.decode, raw)
|
||||
if not ok then
|
||||
redis.call('HDEL', KEYS[6], id)
|
||||
return nil
|
||||
end
|
||||
return info
|
||||
end
|
||||
|
||||
local function remove_response(id)
|
||||
local response_key = response_prefix .. id
|
||||
local info = read_info(id)
|
||||
local metadata = nil
|
||||
local raw = redis.call('GET', response_key)
|
||||
if raw then
|
||||
local ok, decoded = pcall(cjson.decode, raw)
|
||||
if ok then
|
||||
metadata = decoded
|
||||
end
|
||||
end
|
||||
|
||||
local stored_conversation_id = info and info.conversationId or metadata and metadata.conversationId
|
||||
local stored_account_id = info and info.accountId or metadata and metadata.accountId
|
||||
local metadata_bytes = info and tonumber(info.metadataBytes or 0)
|
||||
or redis.call('STRLEN', response_key)
|
||||
|
||||
redis.call('UNLINK', response_key)
|
||||
redis.call('ZREM', KEYS[4], id)
|
||||
redis.call('HDEL', KEYS[6], id)
|
||||
redis.call('ZREM', KEYS[2], id)
|
||||
redis.call('ZREM', KEYS[3], id)
|
||||
decrement_bytes(metadata_bytes)
|
||||
if stored_conversation_id then
|
||||
redis.call('ZREM', conversation_prefix .. stored_conversation_id, id)
|
||||
end
|
||||
if stored_account_id then
|
||||
redis.call('ZREM', account_prefix .. stored_account_id, id)
|
||||
end
|
||||
end
|
||||
|
||||
local function trim_index(key, max_members)
|
||||
local excess = redis.call('ZCARD', key) - max_members
|
||||
if excess <= 0 then
|
||||
return
|
||||
end
|
||||
local ids = redis.call('ZRANGE', key, 0, excess - 1)
|
||||
for _, id in ipairs(ids) do
|
||||
remove_response(id)
|
||||
end
|
||||
end
|
||||
|
||||
local previous_info = read_info(response_id)
|
||||
if previous_info then
|
||||
if previous_info.conversationId and previous_info.conversationId ~= conversation_id then
|
||||
redis.call('ZREM', conversation_prefix .. previous_info.conversationId, response_id)
|
||||
end
|
||||
if previous_info.accountId and previous_info.accountId ~= account_id then
|
||||
redis.call('ZREM', account_prefix .. previous_info.accountId, response_id)
|
||||
end
|
||||
end
|
||||
local previous_bytes = previous_info and tonumber(previous_info.metadataBytes or 0)
|
||||
or redis.call('STRLEN', KEYS[1])
|
||||
local metadata_bytes = string.len(metadata_json)
|
||||
|
||||
redis.call('SET', KEYS[1], metadata_json, 'EXAT', expires_at)
|
||||
redis.call('ZADD', KEYS[2], score, response_id)
|
||||
redis.call('EXPIRE', KEYS[2], ttl_seconds)
|
||||
redis.call('ZADD', KEYS[3], score, response_id)
|
||||
redis.call('EXPIRE', KEYS[3], ttl_seconds)
|
||||
redis.call('ZADD', KEYS[4], score, response_id)
|
||||
redis.call('EXPIRE', KEYS[4], ttl_seconds)
|
||||
redis.call('INCRBY', KEYS[5], metadata_bytes - previous_bytes)
|
||||
redis.call('EXPIRE', KEYS[5], ttl_seconds)
|
||||
redis.call('HSET', KEYS[6], response_id, cjson.encode({
|
||||
conversationId = conversation_id,
|
||||
accountId = account_id,
|
||||
metadataBytes = metadata_bytes,
|
||||
}))
|
||||
redis.call('EXPIRE', KEYS[6], ttl_seconds)
|
||||
|
||||
local stale_conversation_ids = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', cutoff_ms)
|
||||
for _, id in ipairs(stale_conversation_ids) do
|
||||
remove_response(id)
|
||||
end
|
||||
local stale_account_ids = redis.call(
|
||||
'ZRANGEBYSCORE', KEYS[3], '-inf', cutoff_ms, 'LIMIT', 0, stale_cleanup_batch
|
||||
)
|
||||
for _, id in ipairs(stale_account_ids) do
|
||||
remove_response(id)
|
||||
end
|
||||
local stale_global_ids = redis.call(
|
||||
'ZRANGEBYSCORE', KEYS[4], '-inf', cutoff_ms, 'LIMIT', 0, stale_cleanup_batch
|
||||
)
|
||||
for _, id in ipairs(stale_global_ids) do
|
||||
remove_response(id)
|
||||
end
|
||||
|
||||
trim_index(KEYS[2], max_conversation)
|
||||
trim_index(KEYS[3], max_account)
|
||||
trim_index(KEYS[4], max_global)
|
||||
|
||||
while tonumber(redis.call('GET', KEYS[5]) or '0') > max_global_bytes do
|
||||
local oldest = redis.call('ZRANGE', KEYS[4], 0, 0)
|
||||
if #oldest == 0 then
|
||||
redis.call('DEL', KEYS[5])
|
||||
break
|
||||
end
|
||||
remove_response(oldest[1])
|
||||
end
|
||||
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const FORGET_SCRIPT: &str = r#"
|
||||
local response_id = ARGV[1]
|
||||
local response_prefix = ARGV[2]
|
||||
local conversation_prefix = ARGV[3]
|
||||
local account_prefix = ARGV[4]
|
||||
local response_key = response_prefix .. response_id
|
||||
local info_json = redis.call('HGET', KEYS[3], response_id)
|
||||
local metadata_json = redis.call('GET', response_key)
|
||||
if not info_json and not metadata_json then
|
||||
return 0
|
||||
end
|
||||
|
||||
local info = nil
|
||||
if info_json then
|
||||
local ok, decoded = pcall(cjson.decode, info_json)
|
||||
if ok then
|
||||
info = decoded
|
||||
end
|
||||
end
|
||||
local metadata = nil
|
||||
if metadata_json then
|
||||
local ok, decoded = pcall(cjson.decode, metadata_json)
|
||||
if ok then
|
||||
metadata = decoded
|
||||
end
|
||||
end
|
||||
local conversation_id = info and info.conversationId or metadata and metadata.conversationId
|
||||
local account_id = info and info.accountId or metadata and metadata.accountId
|
||||
local metadata_bytes = info and tonumber(info.metadataBytes or 0)
|
||||
or redis.call('STRLEN', response_key)
|
||||
|
||||
redis.call('UNLINK', response_key)
|
||||
redis.call('ZREM', KEYS[1], response_id)
|
||||
redis.call('HDEL', KEYS[3], response_id)
|
||||
if metadata_bytes > 0 then
|
||||
local remaining = redis.call('DECRBY', KEYS[2], metadata_bytes)
|
||||
if remaining <= 0 then
|
||||
redis.call('DEL', KEYS[2])
|
||||
end
|
||||
end
|
||||
if conversation_id then
|
||||
redis.call('ZREM', conversation_prefix .. conversation_id, response_id)
|
||||
end
|
||||
if account_id then
|
||||
redis.call('ZREM', account_prefix .. account_id, response_id)
|
||||
end
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const FORGET_ACCOUNT_SCRIPT: &str = r#"
|
||||
local account_id = ARGV[1]
|
||||
local response_prefix = ARGV[2]
|
||||
local conversation_prefix = ARGV[3]
|
||||
local account_prefix = ARGV[4]
|
||||
local account_key = account_prefix .. account_id
|
||||
local response_ids = redis.call('ZRANGE', account_key, 0, -1)
|
||||
|
||||
local function decrement_bytes(size)
|
||||
if size <= 0 then
|
||||
return
|
||||
end
|
||||
local remaining = redis.call('DECRBY', KEYS[3], size)
|
||||
if remaining <= 0 then
|
||||
redis.call('DEL', KEYS[3])
|
||||
end
|
||||
end
|
||||
|
||||
for _, response_id in ipairs(response_ids) do
|
||||
local response_key = response_prefix .. response_id
|
||||
local info_json = redis.call('HGET', KEYS[4], response_id)
|
||||
local metadata_json = redis.call('GET', response_key)
|
||||
local info = nil
|
||||
if info_json then
|
||||
local ok, decoded = pcall(cjson.decode, info_json)
|
||||
if ok then
|
||||
info = decoded
|
||||
end
|
||||
end
|
||||
local metadata = nil
|
||||
if metadata_json then
|
||||
local ok, decoded = pcall(cjson.decode, metadata_json)
|
||||
if ok then
|
||||
metadata = decoded
|
||||
end
|
||||
end
|
||||
local stored_account_id = info and info.accountId or metadata and metadata.accountId
|
||||
if stored_account_id == account_id then
|
||||
local conversation_id = info and info.conversationId or metadata and metadata.conversationId
|
||||
local metadata_bytes = info and tonumber(info.metadataBytes or 0)
|
||||
or redis.call('STRLEN', response_key)
|
||||
redis.call('UNLINK', response_key)
|
||||
redis.call('ZREM', KEYS[2], response_id)
|
||||
redis.call('HDEL', KEYS[4], response_id)
|
||||
decrement_bytes(metadata_bytes)
|
||||
if conversation_id then
|
||||
redis.call('ZREM', conversation_prefix .. conversation_id, response_id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
redis.call('DEL', account_key)
|
||||
return #response_ids
|
||||
"#;
|
||||
|
||||
/// Redis 会话亲和性存储。
|
||||
#[derive(Clone)]
|
||||
pub struct RedisSessionAffinityStore {
|
||||
redis: RedisConnection,
|
||||
}
|
||||
|
||||
/// Redis 会话亲和性存储错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum RedisSessionAffinityStoreError {
|
||||
#[error("Redis session affinity operation failed: {0}")]
|
||||
Redis(#[from] redis::RedisError),
|
||||
#[error("invalid Redis session affinity value: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("session affinity metadata is too large: {bytes} bytes exceeds {max} bytes")]
|
||||
MetadataTooLarge { bytes: usize, max: usize },
|
||||
}
|
||||
|
||||
pub type RedisSessionAffinityStoreResult<T> = Result<T, RedisSessionAffinityStoreError>;
|
||||
|
||||
impl RedisSessionAffinityStore {
|
||||
pub fn new(redis: RedisConnection) -> Self {
|
||||
Self { redis }
|
||||
}
|
||||
|
||||
pub async fn upsert(
|
||||
&self,
|
||||
response_id: &str,
|
||||
entry: &SessionAffinityEntry,
|
||||
ttl: Duration,
|
||||
) -> RedisSessionAffinityStoreResult<()> {
|
||||
let expires_at = entry
|
||||
.created_at
|
||||
.checked_add_signed(ttl)
|
||||
.unwrap_or(DateTime::<Utc>::MAX_UTC);
|
||||
if expires_at <= Utc::now() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let metadata_json = serde_json::to_string(entry)?;
|
||||
if metadata_json.len() > MAX_AFFINITY_METADATA_BYTES {
|
||||
return Err(RedisSessionAffinityStoreError::MetadataTooLarge {
|
||||
bytes: metadata_json.len(),
|
||||
max: MAX_AFFINITY_METADATA_BYTES,
|
||||
});
|
||||
}
|
||||
|
||||
let ttl_seconds = ttl.num_seconds().max(1);
|
||||
let cutoff_ms = Utc::now().timestamp_millis() - ttl.num_milliseconds();
|
||||
let mut connection = self.redis.manager();
|
||||
let _: i64 = redis::Script::new(UPSERT_SCRIPT)
|
||||
.key(self.response_key(response_id))
|
||||
.key(self.conversation_key(&entry.conversation_id))
|
||||
.key(self.account_key(&entry.account_id))
|
||||
.key(self.global_index_key())
|
||||
.key(self.global_bytes_key())
|
||||
.key(self.global_info_key())
|
||||
.arg(response_id)
|
||||
.arg(metadata_json)
|
||||
.arg(entry.created_at.timestamp_millis())
|
||||
.arg(expires_at.timestamp())
|
||||
.arg(ttl_seconds)
|
||||
.arg(cutoff_ms)
|
||||
.arg(self.response_key_prefix())
|
||||
.arg(self.conversation_key_prefix())
|
||||
.arg(self.account_key_prefix())
|
||||
.arg(MAX_CONVERSATION_AFFINITIES)
|
||||
.arg(MAX_ACCOUNT_AFFINITIES)
|
||||
.arg(MAX_GLOBAL_AFFINITIES)
|
||||
.arg(MAX_GLOBAL_AFFINITY_BYTES)
|
||||
.arg(GLOBAL_STALE_CLEANUP_BATCH)
|
||||
.invoke_async(&mut connection)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get(
|
||||
&self,
|
||||
response_id: &str,
|
||||
now: DateTime<Utc>,
|
||||
ttl: Duration,
|
||||
) -> RedisSessionAffinityStoreResult<Option<SessionAffinityEntry>> {
|
||||
let mut connection = self.redis.manager();
|
||||
let value: Option<String> = connection.get(self.response_key(response_id)).await?;
|
||||
let Some(value) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let entry: SessionAffinityEntry = serde_json::from_str(&value)?;
|
||||
if entry
|
||||
.created_at
|
||||
.checked_add_signed(ttl)
|
||||
.is_none_or(|expires_at| expires_at <= now)
|
||||
{
|
||||
self.forget(response_id).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
pub async fn latest_by_conversation(
|
||||
&self,
|
||||
conversation_id: &str,
|
||||
max_age: Option<Duration>,
|
||||
variant_hash: Option<&str>,
|
||||
now: DateTime<Utc>,
|
||||
ttl: Duration,
|
||||
) -> RedisSessionAffinityStoreResult<Option<(String, SessionAffinityEntry)>> {
|
||||
let conversation_key = self.conversation_key(conversation_id);
|
||||
let mut connection = self.redis.manager();
|
||||
let response_ids: Vec<String> = redis::cmd("ZREVRANGE")
|
||||
.arg(&conversation_key)
|
||||
.arg(0)
|
||||
.arg(MAX_CONVERSATION_AFFINITIES.saturating_sub(1))
|
||||
.query_async(&mut connection)
|
||||
.await?;
|
||||
if response_ids.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let response_keys = response_ids
|
||||
.iter()
|
||||
.map(|response_id| self.response_key(response_id))
|
||||
.collect::<Vec<_>>();
|
||||
let values: Vec<Option<String>> = redis::cmd("MGET")
|
||||
.arg(response_keys)
|
||||
.query_async(&mut connection)
|
||||
.await?;
|
||||
let mut stale_ids = Vec::new();
|
||||
let mut result = None;
|
||||
for (response_id, value) in response_ids.into_iter().zip(values) {
|
||||
let Some(value) = value else {
|
||||
stale_ids.push(response_id);
|
||||
continue;
|
||||
};
|
||||
let entry: SessionAffinityEntry = serde_json::from_str(&value)?;
|
||||
if entry.conversation_id != conversation_id
|
||||
|| entry
|
||||
.created_at
|
||||
.checked_add_signed(ttl)
|
||||
.is_none_or(|expires_at| expires_at <= now)
|
||||
{
|
||||
stale_ids.push(response_id);
|
||||
continue;
|
||||
}
|
||||
if variant_hash.is_some_and(|expected| entry.variant_hash.as_deref() != Some(expected))
|
||||
|| max_age
|
||||
.is_some_and(|max_age| now.signed_duration_since(entry.created_at) > max_age)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
result = Some((response_id, entry));
|
||||
break;
|
||||
}
|
||||
if !stale_ids.is_empty() {
|
||||
let _: usize = connection.zrem(conversation_key, stale_ids).await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn forget(&self, response_id: &str) -> RedisSessionAffinityStoreResult<bool> {
|
||||
let mut connection = self.redis.manager();
|
||||
let forgotten: i64 = redis::Script::new(FORGET_SCRIPT)
|
||||
.key(self.global_index_key())
|
||||
.key(self.global_bytes_key())
|
||||
.key(self.global_info_key())
|
||||
.arg(response_id)
|
||||
.arg(self.response_key_prefix())
|
||||
.arg(self.conversation_key_prefix())
|
||||
.arg(self.account_key_prefix())
|
||||
.invoke_async(&mut connection)
|
||||
.await?;
|
||||
Ok(forgotten > 0)
|
||||
}
|
||||
|
||||
pub async fn forget_account(&self, account_id: &str) -> RedisSessionAffinityStoreResult<u64> {
|
||||
let mut connection = self.redis.manager();
|
||||
let forgotten: u64 = redis::Script::new(FORGET_ACCOUNT_SCRIPT)
|
||||
.key(self.account_key(account_id))
|
||||
.key(self.global_index_key())
|
||||
.key(self.global_bytes_key())
|
||||
.key(self.global_info_key())
|
||||
.arg(account_id)
|
||||
.arg(self.response_key_prefix())
|
||||
.arg(self.conversation_key_prefix())
|
||||
.arg(self.account_key_prefix())
|
||||
.invoke_async(&mut connection)
|
||||
.await?;
|
||||
Ok(forgotten)
|
||||
}
|
||||
|
||||
fn response_key(&self, response_id: &str) -> String {
|
||||
format!("{}{response_id}", self.response_key_prefix())
|
||||
}
|
||||
|
||||
fn conversation_key(&self, conversation_id: &str) -> String {
|
||||
format!("{}{conversation_id}", self.conversation_key_prefix())
|
||||
}
|
||||
|
||||
fn account_key(&self, account_id: &str) -> String {
|
||||
format!("{}{account_id}", self.account_key_prefix())
|
||||
}
|
||||
|
||||
fn global_index_key(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:global:index"))
|
||||
}
|
||||
|
||||
fn global_bytes_key(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:global:bytes"))
|
||||
}
|
||||
|
||||
fn global_info_key(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:global:info"))
|
||||
}
|
||||
|
||||
fn response_key_prefix(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:resp:"))
|
||||
}
|
||||
|
||||
fn conversation_key_prefix(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:conv:"))
|
||||
}
|
||||
|
||||
fn account_key_prefix(&self) -> String {
|
||||
self.redis.key(&format!("{AFFINITY_VERSION}:account:"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//! 会话亲和领域类型。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::upstream::openai::protocol::responses::PreviousResponseScope;
|
||||
|
||||
/// 会话亲和性条目。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SessionAffinityEntry {
|
||||
pub account_id: String,
|
||||
pub conversation_id: String,
|
||||
pub turn_state: Option<String>,
|
||||
pub instructions_hash: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub function_call_ids: Vec<String>,
|
||||
pub variant_hash: Option<String>,
|
||||
pub continuation_scope: PreviousResponseScope,
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
//! 请求级账号候选快照与尝试账本。
|
||||
|
||||
use std::collections::{BTreeSet, VecDeque};
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::time::{Duration, Instant, timeout_at};
|
||||
|
||||
use crate::fleet::pool::{
|
||||
AccountAcquireRequest, AccountCandidateLease, AccountLease, AccountPoolService,
|
||||
};
|
||||
|
||||
const ACCOUNT_CANDIDATE_WAIT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// 请求开始时冻结的完整候选集,以及候选的最终处理状态。
|
||||
pub(in crate::dispatch) struct AccountAttemptLedger {
|
||||
model: String,
|
||||
candidate_ids: Vec<String>,
|
||||
pending: VecDeque<String>,
|
||||
busy: VecDeque<String>,
|
||||
attempted: BTreeSet<String>,
|
||||
state_excluded: BTreeSet<String>,
|
||||
busy_wait_deadline: Instant,
|
||||
}
|
||||
|
||||
impl AccountAttemptLedger {
|
||||
pub(in crate::dispatch) async fn freeze(
|
||||
account_pool: &AccountPoolService,
|
||||
request: &AccountAcquireRequest,
|
||||
) -> Self {
|
||||
let candidate_ids = account_pool.candidate_snapshot(request).await;
|
||||
Self {
|
||||
model: request.model.clone(),
|
||||
pending: candidate_ids.iter().cloned().collect(),
|
||||
candidate_ids,
|
||||
busy: VecDeque::new(),
|
||||
attempted: BTreeSet::new(),
|
||||
state_excluded: BTreeSet::new(),
|
||||
busy_wait_deadline: Instant::now() + ACCOUNT_CANDIDATE_WAIT_TIMEOUT,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn acquire_next(
|
||||
&mut self,
|
||||
account_pool: &AccountPoolService,
|
||||
) -> Option<AccountLease> {
|
||||
loop {
|
||||
while let Some(account_id) = self.pending.pop_front() {
|
||||
match account_pool
|
||||
.acquire_candidate(&self.model, &account_id, Utc::now())
|
||||
.await
|
||||
{
|
||||
AccountCandidateLease::Acquired(lease) => {
|
||||
self.attempted.insert(account_id);
|
||||
return Some(*lease);
|
||||
}
|
||||
AccountCandidateLease::Busy => self.busy.push_back(account_id),
|
||||
AccountCandidateLease::Unavailable => {
|
||||
self.state_excluded.insert(account_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.busy.is_empty() {
|
||||
debug_assert_eq!(
|
||||
self.candidate_ids.len(),
|
||||
self.attempted.len() + self.state_excluded.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
if timeout_at(
|
||||
self.busy_wait_deadline,
|
||||
account_pool.wait_for_candidate_change(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
candidate_count = self.candidate_ids.len(),
|
||||
attempted_count = self.attempted.len(),
|
||||
busy_count = self.busy.len(),
|
||||
timeout_ms = ACCOUNT_CANDIDATE_WAIT_TIMEOUT.as_millis(),
|
||||
"timed out waiting for a busy account candidate"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
std::mem::swap(&mut self.pending, &mut self.busy);
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn candidate_count(&self) -> usize {
|
||||
self.candidate_ids.len()
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn attempted_count(&self) -> usize {
|
||||
self.attempted.len()
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn state_excluded_count(&self) -> usize {
|
||||
self.state_excluded.len()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,721 @@
|
||||
//! Shared upstream error classification for dispatch routes.
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
recovery::exhaustion::{ExhaustedAccount, ExhaustedAccountKind, ExhaustedAccountRef},
|
||||
stream::sse_failure::{sse_failure_error_body, stream_failure_http_status},
|
||||
},
|
||||
upstream::openai::{
|
||||
protocol::{responses::ResponsesSseFailure, sse::SseError},
|
||||
transport::{CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DispatchErrorMetadata {
|
||||
pub failure_class: DispatchFailureClass,
|
||||
pub exhausted_count: Option<usize>,
|
||||
pub upstream_error: Option<String>,
|
||||
pub upstream_status: Option<u16>,
|
||||
pub diagnostics: Option<CodexUpstreamDiagnostics>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) enum DispatchFailureClass {
|
||||
NoAvailableAccounts,
|
||||
QuotaExhausted,
|
||||
RateLimited,
|
||||
Expired,
|
||||
Disabled,
|
||||
Banned,
|
||||
CloudflareChallenge,
|
||||
CloudflarePathBlocked,
|
||||
ModelUnsupported,
|
||||
UpstreamUnavailable,
|
||||
ContinuationBusy,
|
||||
HistoryUnavailable,
|
||||
Upstream,
|
||||
InvalidSse,
|
||||
MissingCompleted,
|
||||
EmptyUpstreamResponse,
|
||||
ResponseFailed,
|
||||
}
|
||||
|
||||
impl DispatchFailureClass {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::NoAvailableAccounts => "no_available_accounts",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Expired => "expired",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Banned => "banned",
|
||||
Self::CloudflareChallenge => "cloudflare_challenge",
|
||||
Self::CloudflarePathBlocked => "cloudflare_path_blocked",
|
||||
Self::ModelUnsupported => "model_unsupported",
|
||||
Self::UpstreamUnavailable => "upstream_unavailable",
|
||||
Self::ContinuationBusy => "continuation_busy",
|
||||
Self::HistoryUnavailable => "history_unavailable",
|
||||
Self::Upstream => "upstream",
|
||||
Self::InvalidSse => "invalid_sse",
|
||||
Self::MissingCompleted => "missing_completed",
|
||||
Self::EmptyUpstreamResponse => "empty_upstream_response",
|
||||
Self::ResponseFailed => "response_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DispatchErrorMetadata {
|
||||
pub(crate) fn no_available_accounts() -> Self {
|
||||
Self {
|
||||
failure_class: DispatchFailureClass::NoAvailableAccounts,
|
||||
exhausted_count: None,
|
||||
upstream_error: None,
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn exhausted_ref(exhausted: ExhaustedAccountRef<'_>) -> Self {
|
||||
Self {
|
||||
failure_class: DispatchFailureClass::from(exhausted.kind),
|
||||
exhausted_count: Some(exhausted.count),
|
||||
upstream_error: Some(exhausted.upstream_error.to_string()),
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream(error: &CodexClientError) -> Self {
|
||||
Self {
|
||||
failure_class: DispatchFailureClass::Upstream,
|
||||
exhausted_count: None,
|
||||
upstream_error: Some(upstream_error_body(error)),
|
||||
upstream_status: match error {
|
||||
CodexClientError::Upstream { status, .. } => Some(status.as_u16()),
|
||||
_ => None,
|
||||
},
|
||||
diagnostics: upstream_error_diagnostics(error).cloned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn simple(failure_class: DispatchFailureClass) -> Self {
|
||||
Self {
|
||||
failure_class,
|
||||
exhausted_count: None,
|
||||
upstream_error: None,
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ExhaustedAccountKind> for DispatchFailureClass {
|
||||
fn from(kind: ExhaustedAccountKind) -> Self {
|
||||
match kind {
|
||||
ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted,
|
||||
ExhaustedAccountKind::RateLimited => Self::RateLimited,
|
||||
ExhaustedAccountKind::Expired => Self::Expired,
|
||||
ExhaustedAccountKind::Disabled => Self::Disabled,
|
||||
ExhaustedAccountKind::Banned => Self::Banned,
|
||||
ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge,
|
||||
ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked,
|
||||
ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported,
|
||||
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn insert_dispatch_error_metadata(
|
||||
object: &mut Map<String, Value>,
|
||||
metadata: DispatchErrorMetadata,
|
||||
) {
|
||||
object.insert(
|
||||
"failureClass".to_string(),
|
||||
Value::String(metadata.failure_class.as_str().to_string()),
|
||||
);
|
||||
if let Some(count) = metadata.exhausted_count {
|
||||
object.insert("exhaustedCount".to_string(), json!(count));
|
||||
}
|
||||
if let Some(error) = metadata.upstream_error {
|
||||
object.insert("upstreamError".to_string(), Value::String(error));
|
||||
}
|
||||
if let Some(status) = metadata.upstream_status {
|
||||
object.insert("upstreamStatus".to_string(), json!(status));
|
||||
}
|
||||
if let Some(diagnostics) = metadata.diagnostics {
|
||||
insert_upstream_diagnostics_metadata(object, &diagnostics);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn insert_upstream_diagnostics_metadata(
|
||||
object: &mut Map<String, Value>,
|
||||
diagnostics: &CodexUpstreamDiagnostics,
|
||||
) {
|
||||
if diagnostics.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(status_code) = diagnostics.status_code {
|
||||
object.insert("upstreamStatus".to_string(), json!(status_code));
|
||||
}
|
||||
if let Some(request_id) = &diagnostics.request_id {
|
||||
object.insert(
|
||||
"upstreamRequestId".to_string(),
|
||||
Value::String(request_id.clone()),
|
||||
);
|
||||
}
|
||||
if !diagnostics.trace_headers.is_empty() {
|
||||
object.insert(
|
||||
"upstreamTraceHeaders".to_string(),
|
||||
json!(diagnostics.trace_headers),
|
||||
);
|
||||
}
|
||||
if let Some(cf_ray) = diagnostics.cf_ray() {
|
||||
object.insert("cfRay".to_string(), Value::String(cf_ray.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_diagnostics(
|
||||
error: &CodexClientError,
|
||||
) -> Option<&CodexUpstreamDiagnostics> {
|
||||
match error {
|
||||
CodexClientError::Upstream { diagnostics, .. } => Some(diagnostics),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { status, .. } if status_code_is_rate_limited(status.as_u16())
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_retryable_upstream_5xx_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { status, .. }
|
||||
if status_code_is_transient_upstream(status.as_u16())
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_retryable_account_transport_error(error: &CodexClientError) -> bool {
|
||||
match error {
|
||||
CodexClientError::Http(error) => error.is_connect() || error.is_timeout(),
|
||||
CodexClientError::StreamIdleTimeout { .. } => true,
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::Transport(_)
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::SendTimeout { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ClosedBeforeTerminal
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::InitialEventTimeout { .. },
|
||||
) => true,
|
||||
_ => is_retryable_upstream_5xx_error(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { status, .. } if status_code_is_quota_exhausted(status.as_u16())
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_model_unsupported_upstream_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { status, body, .. }
|
||||
if status.is_client_error()
|
||||
&& !matches!(status.as_u16(), 401 | 402 | 403 | 404 | 429)
|
||||
&& is_model_unsupported_signal(body)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_history_recovery_upstream_error(error: &CodexClientError) -> bool {
|
||||
upstream_error_code(error).is_some_and(|code| is_history_recovery_code(&code))
|
||||
|| matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable { reason }
|
||||
) if !matches!(
|
||||
reason,
|
||||
crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_continuation_busy_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
reason: crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_body(error: &CodexClientError) -> String {
|
||||
match error {
|
||||
CodexClientError::Upstream { body, .. } => body.clone(),
|
||||
error => error.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_set_cookie_headers(error: &CodexClientError) -> &[String] {
|
||||
match error {
|
||||
CodexClientError::Upstream {
|
||||
set_cookie_headers, ..
|
||||
} => set_cookie_headers,
|
||||
_ => &[],
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_model_unsupported_signal(value: &str) -> bool {
|
||||
let value = value.to_ascii_lowercase();
|
||||
value.contains("model_not_supported")
|
||||
|| value.contains("model_not_available")
|
||||
|| (value.contains("model")
|
||||
&& (value.contains("not supported")
|
||||
|| value.contains("not available")
|
||||
|| value.contains("not_supported")
|
||||
|| value.contains("not_available")))
|
||||
}
|
||||
|
||||
pub(crate) fn is_history_recovery_code(code: &str) -> bool {
|
||||
matches!(
|
||||
code,
|
||||
"previous_response_not_found"
|
||||
| "invalid_encrypted_content"
|
||||
| "missing_tool_output"
|
||||
| "no_tool_output"
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_code(error: &CodexClientError) -> Option<String> {
|
||||
let CodexClientError::Upstream { body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
let value = serde_json::from_str::<Value>(body).ok()?;
|
||||
value
|
||||
.pointer("/response/error/code")
|
||||
.or_else(|| value.pointer("/error/code"))
|
||||
.or_else(|| value.get("code"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
pub(crate) fn rate_limit_cooldown_until(
|
||||
error: &CodexClientError,
|
||||
now: DateTime<Utc>,
|
||||
) -> DateTime<Utc> {
|
||||
let retry_after_seconds = match error {
|
||||
CodexClientError::Upstream {
|
||||
retry_after_seconds,
|
||||
..
|
||||
} => retry_after_seconds.unwrap_or(60),
|
||||
_ => 60,
|
||||
};
|
||||
now + Duration::seconds(retry_after_seconds.min(i64::MAX as u64) as i64)
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_http_status(error: &CodexClientError) -> u16 {
|
||||
match error {
|
||||
CodexClientError::Upstream { status, .. } => status.as_u16(),
|
||||
_ => 502,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn backend_transport_name(transport: CodexBackendTransport) -> &'static str {
|
||||
match transport {
|
||||
CodexBackendTransport::HttpSse => "http_sse",
|
||||
CodexBackendTransport::WebSocket => "websocket",
|
||||
}
|
||||
}
|
||||
|
||||
fn status_code_is_rate_limited(status_code: u16) -> bool {
|
||||
status_code == 429
|
||||
}
|
||||
|
||||
fn status_code_is_quota_exhausted(status_code: u16) -> bool {
|
||||
status_code == 402
|
||||
}
|
||||
|
||||
fn status_code_is_transient_upstream(status_code: u16) -> bool {
|
||||
matches!(status_code, 500..=599)
|
||||
}
|
||||
/// Responses 调度错误。
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ResponseDispatchError {
|
||||
#[error("no active account is available")]
|
||||
NoActiveAccount,
|
||||
#[error("all accounts exhausted by quota")]
|
||||
QuotaExhausted {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by rate limit")]
|
||||
RateLimited {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by expired auth")]
|
||||
Expired {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by disabled auth")]
|
||||
Disabled {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by banned auth")]
|
||||
Banned {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
status_code: u16,
|
||||
},
|
||||
#[error("all accounts exhausted by Cloudflare challenge")]
|
||||
CloudflareChallenge {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by Cloudflare path-block")]
|
||||
CloudflarePathBlocked {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all accounts exhausted by unsupported model")]
|
||||
ModelUnsupported {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all account candidates failed with transient upstream errors")]
|
||||
UpstreamUnavailable {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("the previous response connection is busy; retry the request")]
|
||||
ContinuationBusy,
|
||||
#[error("previous response context is unavailable: {upstream_error}")]
|
||||
HistoryUnavailable { upstream_error: String },
|
||||
#[error("upstream request failed: {0}")]
|
||||
Upstream(#[from] CodexClientError),
|
||||
#[error("invalid upstream SSE response: {0}")]
|
||||
InvalidSse(#[from] SseError),
|
||||
#[error("upstream response did not include response.completed")]
|
||||
MissingCompleted,
|
||||
#[error("upstream response did not include visible output")]
|
||||
EmptyUpstreamResponse,
|
||||
#[error("upstream response failed: {0:?}")]
|
||||
Failed(ResponsesSseFailure),
|
||||
}
|
||||
|
||||
impl ResponseDispatchError {
|
||||
pub(crate) fn from_exhausted_account(exhausted: ExhaustedAccount) -> Self {
|
||||
match exhausted.kind {
|
||||
ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::RateLimited => Self::RateLimited {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::Expired => Self::Expired {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::Disabled => Self::Disabled {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::Banned => Self::Banned {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
status_code: exhausted.status_code.unwrap_or(403),
|
||||
},
|
||||
ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn http_status_code(&self) -> u16 {
|
||||
match self {
|
||||
Self::NoActiveAccount => 503,
|
||||
Self::QuotaExhausted { .. } => 429,
|
||||
Self::RateLimited { .. } => 429,
|
||||
Self::Expired { .. } | Self::Disabled { .. } => 401,
|
||||
Self::Banned { status_code, .. } => *status_code,
|
||||
Self::CloudflareChallenge { .. }
|
||||
| Self::CloudflarePathBlocked { .. }
|
||||
| Self::InvalidSse(_)
|
||||
| Self::MissingCompleted
|
||||
| Self::EmptyUpstreamResponse => 502,
|
||||
Self::UpstreamUnavailable { .. } => 503,
|
||||
Self::ContinuationBusy => 503,
|
||||
Self::Failed(failure) => stream_failure_http_status(failure),
|
||||
Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400,
|
||||
Self::Upstream(error) => upstream_error_http_status(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn client_http_status_code(&self) -> u16 {
|
||||
match self {
|
||||
Self::Upstream(_) => client_upstream_http_status_code(self.http_status_code()),
|
||||
_ => self.http_status_code(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metadata(&self) -> DispatchErrorMetadata {
|
||||
match self {
|
||||
Self::NoActiveAccount => DispatchErrorMetadata::no_available_accounts(),
|
||||
Self::QuotaExhausted {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::QuotaExhausted,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::RateLimited {
|
||||
count,
|
||||
upstream_error,
|
||||
} => {
|
||||
Self::exhausted_metadata(ExhaustedAccountKind::RateLimited, *count, upstream_error)
|
||||
}
|
||||
Self::Expired {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(ExhaustedAccountKind::Expired, *count, upstream_error),
|
||||
Self::Disabled {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(ExhaustedAccountKind::Disabled, *count, upstream_error),
|
||||
Self::Banned {
|
||||
count,
|
||||
upstream_error,
|
||||
..
|
||||
} => Self::exhausted_metadata(ExhaustedAccountKind::Banned, *count, upstream_error),
|
||||
Self::CloudflareChallenge {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::CloudflareChallenge,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::CloudflarePathBlocked {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::CloudflarePathBlocked,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::ModelUnsupported {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::ModelUnsupported,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::UpstreamUnavailable {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::UpstreamUnavailable,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::ContinuationBusy => {
|
||||
DispatchErrorMetadata::simple(DispatchFailureClass::ContinuationBusy)
|
||||
}
|
||||
Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata {
|
||||
failure_class: DispatchFailureClass::HistoryUnavailable,
|
||||
exhausted_count: None,
|
||||
upstream_error: Some(upstream_error.clone()),
|
||||
upstream_status: Some(400),
|
||||
diagnostics: None,
|
||||
},
|
||||
Self::Upstream(error) => DispatchErrorMetadata::upstream(error),
|
||||
Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse),
|
||||
Self::MissingCompleted => {
|
||||
DispatchErrorMetadata::simple(DispatchFailureClass::MissingCompleted)
|
||||
}
|
||||
Self::EmptyUpstreamResponse => {
|
||||
DispatchErrorMetadata::simple(DispatchFailureClass::EmptyUpstreamResponse)
|
||||
}
|
||||
Self::Failed(failure) => DispatchErrorMetadata {
|
||||
failure_class: DispatchFailureClass::ResponseFailed,
|
||||
exhausted_count: None,
|
||||
upstream_error: Some(sse_failure_error_body(failure)),
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn exhausted_account(&self) -> Option<ExhaustedAccountRef<'_>> {
|
||||
match self {
|
||||
Self::QuotaExhausted {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::QuotaExhausted,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::RateLimited {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::RateLimited,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::Expired {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::Expired,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::Disabled {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::Disabled,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::Banned {
|
||||
count,
|
||||
upstream_error,
|
||||
..
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::Banned,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::CloudflareChallenge {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::CloudflareChallenge,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::CloudflarePathBlocked {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::CloudflarePathBlocked,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::ModelUnsupported {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::ModelUnsupported,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::UpstreamUnavailable {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::UpstreamUnavailable,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn exhausted_metadata(
|
||||
kind: ExhaustedAccountKind,
|
||||
count: usize,
|
||||
upstream_error: &str,
|
||||
) -> DispatchErrorMetadata {
|
||||
DispatchErrorMetadata::exhausted_ref(ExhaustedAccountRef {
|
||||
kind,
|
||||
count,
|
||||
upstream_error,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn client_upstream_http_status_code(status: u16) -> u16 {
|
||||
match status {
|
||||
400..=499 | 503 => status,
|
||||
_ => 502,
|
||||
}
|
||||
}
|
||||
|
||||
/// Responses live SSE body stream error.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ResponseDispatchStreamError {
|
||||
#[error("upstream stream failed: {0}")]
|
||||
Upstream(#[from] CodexClientError),
|
||||
}
|
||||
|
||||
pub(super) fn dispatch_error_metadata(
|
||||
error: impl std::fmt::Display,
|
||||
stream: bool,
|
||||
compact: bool,
|
||||
transport: Option<&str>,
|
||||
) -> Value {
|
||||
let mut metadata = serde_json::json!({
|
||||
"stream": stream,
|
||||
"failed": true,
|
||||
"errorKind": "dispatch",
|
||||
"error": error.to_string(),
|
||||
});
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return metadata;
|
||||
};
|
||||
if compact {
|
||||
object.insert("compact".to_string(), Value::Bool(true));
|
||||
}
|
||||
if let Some(transport) = transport {
|
||||
object.insert(
|
||||
"transport".to_string(),
|
||||
Value::String(transport.to_string()),
|
||||
);
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
pub(super) fn enrich_response_dispatch_error_metadata(
|
||||
metadata: &mut Value,
|
||||
error: &ResponseDispatchError,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_dispatch_error_metadata(object, error.metadata());
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! 请求调度模块(账号选择、回退、恢复、会话亲和性)。
|
||||
|
||||
pub mod affinity;
|
||||
mod attempts;
|
||||
pub mod errors;
|
||||
mod recording;
|
||||
pub mod recovery;
|
||||
pub(crate) mod service;
|
||||
pub(crate) mod stream;
|
||||
pub mod upstream_call;
|
||||
@@ -0,0 +1,687 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{
|
||||
dispatch::errors::{
|
||||
DispatchErrorMetadata, backend_transport_name, insert_dispatch_error_metadata,
|
||||
insert_upstream_diagnostics_metadata, upstream_error_http_status,
|
||||
},
|
||||
infra::time::elapsed_millis_i64,
|
||||
telemetry::{
|
||||
ops::types::OpsErrorLog,
|
||||
recorder::{
|
||||
DispatchErrorLogRecord, Recorder, enrich_event_route_metadata,
|
||||
enrich_response_request_semantics, enrich_usage_record_identity,
|
||||
event_kind as response_event_kind, reasoning_effort_from_request,
|
||||
record_dispatch_error_event,
|
||||
},
|
||||
usage::types::UsageRecordLevel,
|
||||
},
|
||||
upstream::openai::{
|
||||
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
|
||||
transport::{
|
||||
CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
|
||||
WebSocketPoolDecision,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
use super::{
|
||||
errors::{
|
||||
ResponseDispatchError, dispatch_error_metadata, enrich_response_dispatch_error_metadata,
|
||||
},
|
||||
stream::{
|
||||
live::{LiveResponseStreamContext, latest_response_id},
|
||||
sse_failure::{status_code_for_stream_failure, stream_failure_metadata},
|
||||
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
|
||||
},
|
||||
};
|
||||
|
||||
pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
|
||||
pub(super) recorder: &'a Recorder,
|
||||
pub(super) request_id: &'a str,
|
||||
pub(super) account_id: &'a str,
|
||||
pub(super) account_email: Option<&'a str>,
|
||||
pub(super) route: &'a str,
|
||||
pub(super) model: &'a str,
|
||||
pub(super) started_at: Instant,
|
||||
pub(super) stream: bool,
|
||||
pub(super) transport: CodexBackendTransport,
|
||||
pub(super) request: &'a CodexResponsesRequest,
|
||||
pub(super) error: &'a CodexClientError,
|
||||
pub(super) trace: &'a ResponseDispatchTrace,
|
||||
pub(super) attempt: Option<&'a ResponseDispatchAttempt>,
|
||||
}
|
||||
|
||||
pub(super) struct ResponseStreamFailureEventRecord<'a> {
|
||||
pub(super) recorder: &'a Recorder,
|
||||
pub(super) request_id: &'a str,
|
||||
pub(super) account_id: &'a str,
|
||||
pub(super) route: &'a str,
|
||||
pub(super) model: &'a str,
|
||||
pub(super) requested_model: &'a str,
|
||||
pub(super) started_at: Instant,
|
||||
pub(super) transport: CodexBackendTransport,
|
||||
pub(super) request: &'a CodexResponsesRequest,
|
||||
pub(super) failure: &'a ResponsesSseFailure,
|
||||
pub(super) diagnostics: &'a CodexUpstreamDiagnostics,
|
||||
pub(super) rate_limit_headers: &'a [(String, String)],
|
||||
pub(super) prefetched: &'a [u8],
|
||||
pub(super) trace: &'a ResponseDispatchTrace,
|
||||
pub(super) attempt: &'a ResponseDispatchAttempt,
|
||||
}
|
||||
|
||||
pub(super) struct ResponseDispatchErrorEventRecord<'a> {
|
||||
pub(super) recorder: &'a Recorder,
|
||||
pub(super) request_id: &'a str,
|
||||
pub(super) client_api_key_id: Option<&'a str>,
|
||||
pub(super) account_id: Option<&'a str>,
|
||||
pub(super) route: &'a str,
|
||||
pub(super) model: &'a str,
|
||||
pub(super) started_at: Instant,
|
||||
pub(super) stream: bool,
|
||||
pub(super) compact: bool,
|
||||
pub(super) transport: Option<&'a str>,
|
||||
pub(super) error: &'a ResponseDispatchError,
|
||||
}
|
||||
|
||||
pub(super) struct ResponseDispatchErrorDetails<'a> {
|
||||
pub(super) client_api_key_id: Option<&'a str>,
|
||||
pub(super) account_id: Option<&'a str>,
|
||||
pub(super) stream: bool,
|
||||
pub(super) compact: bool,
|
||||
pub(super) transport: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub(super) async fn record_response_dispatch_error_event(
|
||||
record: ResponseDispatchErrorEventRecord<'_>,
|
||||
) {
|
||||
let mut metadata = dispatch_error_metadata(
|
||||
record.error,
|
||||
record.stream,
|
||||
record.compact,
|
||||
record.transport,
|
||||
);
|
||||
enrich_response_dispatch_error_metadata(&mut metadata, record.error);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
i64::from(record.error.http_status_code()),
|
||||
dispatch_error_client_status_code(record.error),
|
||||
dispatch_error_upstream_status_code(record.error),
|
||||
);
|
||||
}
|
||||
record_dispatch_error_event(DispatchErrorLogRecord {
|
||||
recorder: record.recorder,
|
||||
request_id: record.request_id,
|
||||
client_api_key_id: record.client_api_key_id,
|
||||
provider: Some("openai"),
|
||||
account_id: record.account_id,
|
||||
route: record.route,
|
||||
model: record.model,
|
||||
started_at: record.started_at,
|
||||
status_code: i64::from(record.error.http_status_code()),
|
||||
message: "v1 responses dispatch failed",
|
||||
metadata,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn record_response_upstream_error_event(
|
||||
record: ResponseUpstreamErrorEventRecord<'_>,
|
||||
) {
|
||||
let event_status_code = i64::from(upstream_error_http_status(record.error));
|
||||
let mut metadata = dispatch_error_metadata(
|
||||
record.error,
|
||||
record.stream,
|
||||
false,
|
||||
Some(backend_transport_name(record.transport)),
|
||||
);
|
||||
enrich_event_route_metadata(&mut metadata, record.route);
|
||||
let mut event = OpsErrorLog::new("v1.response", "v1 responses upstream request failed");
|
||||
event.request_id = Some(record.request_id.to_string());
|
||||
event.client_api_key_id = record.request.client_api_key_id.clone();
|
||||
event.provider = Some("openai".to_string());
|
||||
event.account_id = Some(record.account_id.to_string());
|
||||
event.route = Some(record.route.to_string());
|
||||
event.model = Some(record.model.to_string());
|
||||
event.status_code = Some(event_status_code);
|
||||
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
insert_dispatch_error_metadata(object, DispatchErrorMetadata::upstream(record.error));
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
upstream_failure_client_status_code(event_status_code),
|
||||
Some(event_status_code),
|
||||
);
|
||||
insert_response_request_summary_object(object, record.request, record.transport);
|
||||
insert_response_trace_metadata_object(object, record.trace.attempts(), record.attempt);
|
||||
if let Some(account_email) = record
|
||||
.account_email
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
object.insert(
|
||||
"accountEmail".to_string(),
|
||||
Value::String(account_email.to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
enrich_usage_record_identity(
|
||||
&mut metadata,
|
||||
Some(record.request.model()),
|
||||
record.model,
|
||||
record.request.client_ip.as_deref(),
|
||||
record.request.client_user_agent.as_deref(),
|
||||
reasoning_effort_from_request(record.request),
|
||||
record.request.service_tier(),
|
||||
);
|
||||
enrich_response_request_semantics(&mut metadata, record.request);
|
||||
event.metadata = metadata;
|
||||
if let Err(error) = record.recorder.record_error(event).await {
|
||||
tracing::error!(account_id = %record.account_id, error = %error, "failed to record upstream error event");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_prefetched_response_stream_failure_event(
|
||||
record: ResponseStreamFailureEventRecord<'_>,
|
||||
) {
|
||||
let event_status_code = status_code_for_stream_failure(record.failure);
|
||||
let mut metadata = stream_failure_metadata(record.failure, None);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
if record.transport == CodexBackendTransport::WebSocket {
|
||||
object.insert(
|
||||
"transport".to_string(),
|
||||
Value::String("websocket".to_string()),
|
||||
);
|
||||
}
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
200,
|
||||
diagnostics_status_code(record.diagnostics),
|
||||
);
|
||||
insert_response_request_summary_object(object, record.request, record.transport);
|
||||
if record.recorder.captures_body() {
|
||||
object.insert("requestBody".to_string(), json!(record.request));
|
||||
object.insert(
|
||||
"responseBody".to_string(),
|
||||
Value::String(String::from_utf8_lossy(record.prefetched).to_string()),
|
||||
);
|
||||
}
|
||||
insert_upstream_diagnostics_metadata(object, record.diagnostics);
|
||||
insert_response_trace_metadata_object(
|
||||
object,
|
||||
record.trace.attempts(),
|
||||
Some(record.attempt),
|
||||
);
|
||||
}
|
||||
enrich_event_route_metadata(&mut metadata, record.route);
|
||||
enrich_usage_record_identity(
|
||||
&mut metadata,
|
||||
Some(record.requested_model),
|
||||
record.model,
|
||||
record.request.client_ip.as_deref(),
|
||||
record.request.client_user_agent.as_deref(),
|
||||
reasoning_effort_from_request(record.request),
|
||||
record.request.service_tier(),
|
||||
);
|
||||
enrich_response_request_semantics(&mut metadata, record.request);
|
||||
let mut event = OpsErrorLog::new(
|
||||
response_event_kind(record.route),
|
||||
"v1 responses stream failed",
|
||||
);
|
||||
event.request_id = Some(record.request_id.to_string());
|
||||
event.client_api_key_id = record.request.client_api_key_id.clone();
|
||||
event.provider = Some("openai".to_string());
|
||||
event.account_id = Some(record.account_id.to_string());
|
||||
event.route = Some(record.route.to_string());
|
||||
event.model = Some(record.model.to_string());
|
||||
event.status_code = Some(event_status_code);
|
||||
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
||||
if !record.rate_limit_headers.is_empty()
|
||||
&& let Some(object) = metadata.as_object_mut()
|
||||
{
|
||||
object.insert(
|
||||
"rateLimitHeaders".to_string(),
|
||||
serde_json::json!(record.rate_limit_headers),
|
||||
);
|
||||
}
|
||||
event.metadata = metadata;
|
||||
if let Err(error) = record.recorder.record_error(event).await {
|
||||
tracing::error!(account_id = %record.account_id, error = %error, "failed to record prefetched stream error event");
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_stream_metadata_flag(metadata: &mut Value) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
*metadata = serde_json::json!({ "stream": true });
|
||||
return;
|
||||
};
|
||||
object
|
||||
.entry("stream".to_string())
|
||||
.or_insert(Value::Bool(true));
|
||||
}
|
||||
|
||||
fn enrich_live_response_stream_metadata(
|
||||
context: &LiveResponseStreamContext,
|
||||
rate_limit_headers: &[(String, String)],
|
||||
metadata: &mut Value,
|
||||
status_code: i64,
|
||||
body: &str,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
object
|
||||
.entry("transport".to_string())
|
||||
.or_insert_with(|| Value::String(backend_transport_name(context.transport).to_string()));
|
||||
if !rate_limit_headers.is_empty() {
|
||||
object
|
||||
.entry("rateLimitHeaders".to_string())
|
||||
.or_insert_with(|| serde_json::json!(rate_limit_headers));
|
||||
}
|
||||
if let Some(decision) = context.websocket_pool_decision {
|
||||
object
|
||||
.entry("websocketPool".to_string())
|
||||
.or_insert_with(|| decision.metadata_value());
|
||||
}
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
status_code,
|
||||
200,
|
||||
diagnostics_status_code(&context.diagnostics),
|
||||
);
|
||||
insert_response_request_summary_object(object, &context.request, context.transport);
|
||||
insert_upstream_diagnostics_metadata(object, &context.diagnostics);
|
||||
insert_response_trace_metadata_object(object, &context.attempts, Some(&context.attempt));
|
||||
if context.recorder.captures_body() {
|
||||
object
|
||||
.entry("requestBody".to_string())
|
||||
.or_insert_with(|| serde_json::json!(context.request));
|
||||
object
|
||||
.entry("responseBody".to_string())
|
||||
.or_insert_with(|| Value::String(body.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_upstream_diagnostics(
|
||||
metadata: &mut Value,
|
||||
diagnostics: &CodexUpstreamDiagnostics,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_upstream_diagnostics_metadata(object, diagnostics);
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_status_metadata(
|
||||
metadata: &mut Value,
|
||||
event_status_code: i64,
|
||||
client_status_code: i64,
|
||||
upstream_status_code: Option<i64>,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
client_status_code,
|
||||
upstream_status_code,
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_trace_metadata(
|
||||
metadata: &mut Value,
|
||||
trace: &ResponseDispatchTrace,
|
||||
current_attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_response_trace_metadata_object(object, trace.attempts(), current_attempt);
|
||||
}
|
||||
|
||||
fn insert_response_attempt_metadata_object(
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
let Some(attempt) = attempt else {
|
||||
return;
|
||||
};
|
||||
object.insert("attemptIndex".to_string(), json!(attempt.index()));
|
||||
object.insert(
|
||||
"attemptAccountId".to_string(),
|
||||
Value::String(attempt.account_id().to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_response_trace_metadata_object(
|
||||
object: &mut Map<String, Value>,
|
||||
attempts: &[ResponseDispatchAttempt],
|
||||
current_attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
insert_response_attempt_metadata_object(object, current_attempt);
|
||||
object.insert("attemptCount".to_string(), json!(attempts.len()));
|
||||
object.insert(
|
||||
"attempts".to_string(),
|
||||
Value::Array(
|
||||
attempts
|
||||
.iter()
|
||||
.map(|attempt| {
|
||||
json!({
|
||||
"index": attempt.index(),
|
||||
"accountId": attempt.account_id(),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_response_status_metadata_object(
|
||||
object: &mut Map<String, Value>,
|
||||
event_status_code: i64,
|
||||
client_status_code: i64,
|
||||
upstream_status_code: Option<i64>,
|
||||
) {
|
||||
object.insert("eventStatusCode".to_string(), json!(event_status_code));
|
||||
object.insert("clientStatusCode".to_string(), json!(client_status_code));
|
||||
if let Some(upstream_status_code) = upstream_status_code {
|
||||
object.insert(
|
||||
"upstreamStatusCode".to_string(),
|
||||
json!(upstream_status_code),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_response_request_summary_object(
|
||||
object: &mut Map<String, Value>,
|
||||
request: &CodexResponsesRequest,
|
||||
transport: CodexBackendTransport,
|
||||
) {
|
||||
object.insert(
|
||||
"requestSummary".to_string(),
|
||||
response_request_summary(request, transport),
|
||||
);
|
||||
}
|
||||
|
||||
fn response_request_summary(
|
||||
request: &CodexResponsesRequest,
|
||||
transport: CodexBackendTransport,
|
||||
) -> Value {
|
||||
let body = request.body();
|
||||
let input = body.get("input");
|
||||
let tools = body.get("tools");
|
||||
let semantics = request.semantics();
|
||||
json!({
|
||||
"model": request.model(),
|
||||
"stream": request.stream(),
|
||||
"store": request.store(),
|
||||
"compact": semantics.compact,
|
||||
"requestKind": semantics.request_kind,
|
||||
"subagentKind": semantics.subagent_kind,
|
||||
"transport": backend_transport_name(transport),
|
||||
"inputType": json_value_kind(input),
|
||||
"inputItemsCount": input.and_then(Value::as_array).map(Vec::len),
|
||||
"toolsType": json_value_kind(tools),
|
||||
"toolsCount": tools.and_then(Value::as_array).map(Vec::len),
|
||||
"topLevelFields": body.keys().cloned().collect::<Vec<_>>(),
|
||||
"previousResponseIdPresent": request.previous_response_id().is_some(),
|
||||
"serviceTier": request.service_tier(),
|
||||
"localTransport": {
|
||||
"useWebsocket": request.use_websocket,
|
||||
"forceHttpSse": request.force_http_sse,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn json_value_kind(value: Option<&Value>) -> &'static str {
|
||||
match value {
|
||||
None => "missing",
|
||||
Some(Value::Null) => "null",
|
||||
Some(Value::Bool(_)) => "boolean",
|
||||
Some(Value::Number(_)) => "number",
|
||||
Some(Value::String(_)) => "string",
|
||||
Some(Value::Array(_)) => "array",
|
||||
Some(Value::Object(_)) => "object",
|
||||
}
|
||||
}
|
||||
|
||||
fn diagnostics_status_code(diagnostics: &CodexUpstreamDiagnostics) -> Option<i64> {
|
||||
diagnostics.status_code.map(i64::from)
|
||||
}
|
||||
|
||||
fn dispatch_error_upstream_status_code(error: &ResponseDispatchError) -> Option<i64> {
|
||||
match error {
|
||||
ResponseDispatchError::Upstream(error) => {
|
||||
Some(i64::from(upstream_error_http_status(error)))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_error_client_status_code(error: &ResponseDispatchError) -> i64 {
|
||||
i64::from(error.client_http_status_code())
|
||||
}
|
||||
|
||||
fn upstream_failure_client_status_code(upstream_status: i64) -> i64 {
|
||||
i64::from(crate::dispatch::errors::client_upstream_http_status_code(
|
||||
u16::try_from(upstream_status).unwrap_or(502),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn record_live_response_stream_event(
|
||||
context: &LiveResponseStreamContext,
|
||||
status_code: i64,
|
||||
level: UsageRecordLevel,
|
||||
message: &str,
|
||||
mut metadata: Value,
|
||||
rate_limit_headers: &[(String, String)],
|
||||
body: &str,
|
||||
) {
|
||||
let effective_model = context
|
||||
.response_metadata
|
||||
.effective_model
|
||||
.as_deref()
|
||||
.unwrap_or(&context.display_model);
|
||||
ensure_stream_metadata_flag(&mut metadata);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
object.insert(
|
||||
"effectiveModel".to_string(),
|
||||
Value::String(effective_model.to_string()),
|
||||
);
|
||||
object.insert(
|
||||
"modelsEtag".to_string(),
|
||||
context
|
||||
.response_metadata
|
||||
.models_etag
|
||||
.as_ref()
|
||||
.map_or(Value::Null, |etag| Value::String(etag.clone())),
|
||||
);
|
||||
object.insert(
|
||||
"reasoningIncluded".to_string(),
|
||||
Value::Bool(context.response_metadata.reasoning_included),
|
||||
);
|
||||
}
|
||||
enrich_event_route_metadata(&mut metadata, &context.route);
|
||||
enrich_live_response_stream_metadata(
|
||||
context,
|
||||
rate_limit_headers,
|
||||
&mut metadata,
|
||||
status_code,
|
||||
body,
|
||||
);
|
||||
log_live_response_stream_finalized(context, status_code, level, message, &metadata, body);
|
||||
enrich_usage_record_identity(
|
||||
&mut metadata,
|
||||
Some(&context.requested_model),
|
||||
effective_model,
|
||||
context.client_ip.as_deref(),
|
||||
context.request.client_user_agent.as_deref(),
|
||||
reasoning_effort_from_request(&context.request),
|
||||
context.request.service_tier(),
|
||||
);
|
||||
enrich_response_request_semantics(&mut metadata, &context.request);
|
||||
|
||||
if is_success_usage_event(level, status_code) {
|
||||
crate::telemetry::recorder::record_response_event(
|
||||
crate::telemetry::usage::types::ResponseUsageRecord {
|
||||
recorder: &context.recorder,
|
||||
request_id: &context.request_id,
|
||||
client_api_key_id: context.request.client_api_key_id.as_deref(),
|
||||
account_id: &context.account_id,
|
||||
route: &context.route,
|
||||
model: effective_model,
|
||||
requested_model: Some(&context.requested_model),
|
||||
client_ip: context.client_ip.as_deref(),
|
||||
client_user_agent: context.request.client_user_agent.as_deref(),
|
||||
reasoning_effort: reasoning_effort_from_request(&context.request),
|
||||
service_tier: context.request.service_tier(),
|
||||
started_at: context.started_at,
|
||||
status_code,
|
||||
message,
|
||||
metadata,
|
||||
rate_limit_headers,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
let mut event = OpsErrorLog::new(response_event_kind(&context.route), message);
|
||||
event.request_id = Some(context.request_id.clone());
|
||||
event.client_api_key_id = context.request.client_api_key_id.clone();
|
||||
event.provider = Some("openai".to_string());
|
||||
event.account_id = Some(context.account_id.clone());
|
||||
event.route = Some(context.route.clone());
|
||||
event.model = Some(effective_model.to_string());
|
||||
event.status_code = Some(status_code);
|
||||
event.latency_ms = Some(elapsed_millis_i64(context.started_at));
|
||||
event.metadata = metadata;
|
||||
if let Err(error) = context.recorder.record_error(event).await {
|
||||
tracing::error!(account_id = %context.account_id, error = %error, "failed to record live response stream error event");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_success_usage_event(level: UsageRecordLevel, status_code: i64) -> bool {
|
||||
level != UsageRecordLevel::Error && status_code < 400
|
||||
}
|
||||
|
||||
fn log_live_response_stream_finalized(
|
||||
context: &LiveResponseStreamContext,
|
||||
status_code: i64,
|
||||
level: UsageRecordLevel,
|
||||
message: &str,
|
||||
metadata: &Value,
|
||||
body: &str,
|
||||
) {
|
||||
let response_id = metadata_string_field(metadata, "responseId")
|
||||
.map(ToString::to_string)
|
||||
.or_else(|| latest_response_id(body));
|
||||
let first_token_ms = metadata.get("firstTokenMs").and_then(Value::as_i64);
|
||||
let websocket_pool_kind = context
|
||||
.websocket_pool_decision
|
||||
.map(WebSocketPoolDecision::kind);
|
||||
let websocket_pool_reason = context
|
||||
.websocket_pool_decision
|
||||
.and_then(WebSocketPoolDecision::reason);
|
||||
let completed = metadata.get("completed").and_then(Value::as_bool);
|
||||
let failed = metadata.get("failed").and_then(Value::as_bool);
|
||||
let upstream_code = metadata_string_field(metadata, "upstreamCode");
|
||||
let failure_class = metadata_string_field(metadata, "failureClass");
|
||||
let failure_source = metadata_string_field(metadata, "failureSource");
|
||||
let failure_detail = metadata_string_field(metadata, "failureDetail");
|
||||
|
||||
macro_rules! emit_stream_finalized_log {
|
||||
($level:expr) => {
|
||||
tracing::event!(
|
||||
$level,
|
||||
account_id = %context.account_id,
|
||||
request_id = %context.request_id,
|
||||
route = %context.route,
|
||||
model = %context.display_model,
|
||||
status_code,
|
||||
usage_level = ?level,
|
||||
event_message = %message,
|
||||
transport = %backend_transport_name(context.transport),
|
||||
websocket_pool_kind = ?websocket_pool_kind,
|
||||
websocket_pool_reason = ?websocket_pool_reason,
|
||||
response_id = response_id.as_deref().unwrap_or(""),
|
||||
first_token_ms = ?first_token_ms,
|
||||
latency_ms = elapsed_millis_i64(context.started_at),
|
||||
completed = ?completed,
|
||||
failed = ?failed,
|
||||
upstream_code = ?upstream_code,
|
||||
failure_class = ?failure_class,
|
||||
failure_source = ?failure_source,
|
||||
failure_detail = ?failure_detail,
|
||||
"live response stream finalized"
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
match level {
|
||||
UsageRecordLevel::Debug => {
|
||||
emit_stream_finalized_log!(tracing::Level::DEBUG);
|
||||
}
|
||||
UsageRecordLevel::Info => {
|
||||
emit_stream_finalized_log!(tracing::Level::INFO);
|
||||
}
|
||||
UsageRecordLevel::Warn => {
|
||||
emit_stream_finalized_log!(tracing::Level::WARN);
|
||||
}
|
||||
UsageRecordLevel::Error => {
|
||||
emit_stream_finalized_log!(tracing::Level::ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata_string_field<'a>(metadata: &'a Value, field: &str) -> Option<&'a str> {
|
||||
metadata.get(field).and_then(Value::as_str)
|
||||
}
|
||||
|
||||
pub(super) async fn live_response_rate_limit_headers(
|
||||
context: &LiveResponseStreamContext,
|
||||
) -> Vec<(String, String)> {
|
||||
let mut headers = context.rate_limit_headers.clone();
|
||||
if let Some(updates) = &context.rate_limit_header_updates {
|
||||
headers.extend(updates.lock().await.iter().cloned());
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
pub(super) async fn live_response_turn_state(
|
||||
context: &LiveResponseStreamContext,
|
||||
) -> Option<String> {
|
||||
if let Some(update) = &context.turn_state_update {
|
||||
return update.lock().await.clone();
|
||||
}
|
||||
context.turn_state.clone()
|
||||
}
|
||||
|
||||
pub(super) fn insert_first_token_ms(metadata: &mut Value, first_token_ms: Option<i64>) {
|
||||
let Some(first_token_ms) = first_token_ms else {
|
||||
return;
|
||||
};
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
object.insert(
|
||||
"firstTokenMs".to_string(),
|
||||
Value::Number(first_token_ms.into()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_websocket_pool_decision(
|
||||
metadata: &mut Value,
|
||||
decision: Option<WebSocketPoolDecision>,
|
||||
) {
|
||||
let Some(decision) = decision else {
|
||||
return;
|
||||
};
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
object.insert("websocketPool".to_string(), decision.metadata_value());
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user